diff --git a/collectors/cloudstack/ReleaseNotes.txt b/collectors/cloudstack/ReleaseNotes.txt new file mode 100644 index 0000000..a645519 --- /dev/null +++ b/collectors/cloudstack/ReleaseNotes.txt @@ -0,0 +1,7 @@ +Version: 1.1.0 +Date: 16/September/2016 +Notes: New output format for Usage Data + +Version: 1.0.0 +Date: 1/July/2016 +Notes: Initial release of the RCB Cyclops CloudStack collector \ No newline at end of file diff --git a/collectors/cloudstack/bin/cloudstack.jar b/collectors/cloudstack/bin/cloudstack.jar index b12c2d5..2bef9b7 100644 Binary files a/collectors/cloudstack/bin/cloudstack.jar and b/collectors/cloudstack/bin/cloudstack.jar differ diff --git a/collectors/cloudstack/config/cloudstack.conf b/collectors/cloudstack/config/cloudstack.conf index a464957..e5f3e4f 100644 --- a/collectors/cloudstack/config/cloudstack.conf +++ b/collectors/cloudstack/config/cloudstack.conf @@ -2,12 +2,12 @@ ServerHTTPPort=4573 # CloudStack Endpoint and its credentials -CloudStackURL=https://yoururl -CloudStackAPIKey=yourapi -CloudStackSecretKey=yoursecretkey +CloudStackURL=url +CloudStackAPIKey=key +CloudStackSecretKey=secret # If you want to start importing only from this day, specify YYYY-MM-DD, if not comment the line -CloudStackFirstImport=2016-06-29 +CloudStackFirstImport=2016-09-16 # Publisher (RabbitMQ) credentials PublisherHost=localhost diff --git a/collectors/cloudstack/pom.xml b/collectors/cloudstack/pom.xml index b9d34a8..af6b612 100644 --- a/collectors/cloudstack/pom.xml +++ b/collectors/cloudstack/pom.xml @@ -7,7 +7,7 @@ ch.icclab.cyclops.cloudstack cyclops-cloudstack-collector jar - 0.0.1 + 1.1.0 CloudStack collector @@ -43,6 +43,7 @@ org.apache.maven.plugins maven-assembly-plugin + 2.6 jar-with-dependencies diff --git a/collectors/cloudstack/scripts/compile.sh b/collectors/cloudstack/scripts/compile.sh index ddeae2c..c43a2d6 100755 --- a/collectors/cloudstack/scripts/compile.sh +++ b/collectors/cloudstack/scripts/compile.sh @@ -26,4 +26,4 @@ mvn dependency:tree mvn package assembly:single cd target -mv cyclops-cloudstack-collector-0.0.1-jar-with-dependencies.jar ../bin/cloudstack.jar +mv cyclops-cloudstack-collector-1.1.0-jar-with-dependencies.jar ../bin/cloudstack.jar diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java index 7810649..a741470 100644 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java @@ -36,6 +36,6 @@ public class RootEndpoint extends ServerResource { @Get public String root(){ counter.increment(ENDPOINT); - return "RCB CloudStack collector micro service - version 1.0.1"; + return "RCB CloudStack collector micro service - version 1.1.0"; } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/IPUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/IP.java similarity index 91% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/IPUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/IP.java index 1ffa314..3ab801b 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/IPUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/IP.java @@ -24,7 +24,7 @@ * Created on: 14-Oct-15 * Description: POJO object for IP Usage Data (type 2) */ -public class IPUsageData extends UsageData { +public class IP extends UsageData { // Whether source NAT is enabled for the IP address private boolean issourcenat; @@ -50,4 +50,9 @@ public boolean issystem() { public void setIssystem(boolean issystem) { this.issystem = issystem; } + + @Override + protected void additionalMetadata(Map map) { + // nothing to do + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/TemplateAndIsoUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/ISO.java similarity index 84% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/TemplateAndIsoUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/ISO.java index 3196451..7e23676 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/TemplateAndIsoUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/ISO.java @@ -16,7 +16,6 @@ */ package ch.icclab.cyclops.model; -import java.util.HashMap; import java.util.Map; /** @@ -24,7 +23,7 @@ * Created on: 15-Oct-15 * Description: POJO object for Template and ISO Usage Data, both type 7 (Template) and 8 (ISO) */ -public class TemplateAndIsoUsageData extends UsageData { +public class ISO extends UsageData { // Size of the template, or ISO private Long size; @@ -48,4 +47,12 @@ public Long getVirtualsize() { public void setVirtualsize(Long virtualsize) { this.virtualsize = virtualsize; } + + @Override + protected void additionalMetadata(Map map) { + if (size != null) + map.put("size", size); + if (virtualsize != null) + map.put("virtualSize", virtualsize); + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/PolicyOrRuleUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/LoadBalancer.java similarity index 86% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/PolicyOrRuleUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/LoadBalancer.java index 8c4e15e..daf2ce7 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/PolicyOrRuleUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/LoadBalancer.java @@ -24,5 +24,9 @@ * Created on: 15-Oct-15 * Description: POJO object for Load Balancer Policy (type 11) or Port Forwarding Rule Usage Data (type 12) */ -public class PolicyOrRuleUsageData extends UsageData { +public class LoadBalancer extends UsageData { + @Override + protected void additionalMetadata(Map map) { + // nothing to do here + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkOfferingUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkOffering.java similarity index 86% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkOfferingUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkOffering.java index 36725a6..11465ab 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkOfferingUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkOffering.java @@ -24,7 +24,7 @@ * Created on: 15-Oct-15 * Description: POJO object for Network Offering Usage Data (type 13) */ -public class NetworkOfferingUsageData extends UsageData { +public class NetworkOffering extends UsageData { private boolean isdefault; @@ -48,4 +48,11 @@ public String getVirtualmachineid() { public void setVirtualmachineid(String virtualmachineid) { this.virtualmachineid = virtualmachineid; } + + @Override + protected void additionalMetadata(Map map) { + if (virtualmachineid != null) { + map.put("virtualmachineId", virtualmachineid); + } + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkReceived.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkReceived.java new file mode 100755 index 0000000..9ebb909 --- /dev/null +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkReceived.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.model; + +import java.util.Map; + +/** + * Author: Martin Skoviera + * Created on: 15-Oct-15 + * Description: POJO object for Network Usage Data, both type 4 (Sent) and 5 (Received) + */ +public class NetworkReceived extends UsageData { + + // Device type (domain router, external load balancer, etc.) + private String type; + + // Its network ID + private String networkid; + + ///////////////////////////// + // Getters and Setters + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getNetworkid() { + return networkid; + } + + public void setNetworkid(String networkid) { + this.networkid = networkid; + } + + @Override + protected void additionalMetadata(Map map) { + if (type != null) + map.put("type", type); + if (networkid != null) + map.put("networkId", networkid); + } +} diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkSent.java similarity index 83% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkSent.java index 281da59..7d59236 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkSent.java @@ -16,12 +16,14 @@ */ package ch.icclab.cyclops.model; +import java.util.Map; + /** * Author: Martin Skoviera * Created on: 15-Oct-15 * Description: POJO object for Network Usage Data, both type 4 (Sent) and 5 (Received) */ -public class NetworkUsageData extends UsageData { +public class NetworkSent extends UsageData { // Device type (domain router, external load balancer, etc.) private String type; @@ -47,4 +49,12 @@ public String getNetworkid() { public void setNetworkid(String networkid) { this.networkid = networkid; } + + @Override + protected void additionalMetadata(Map map) { + if (type != null) + map.put("type", type); + if (networkid != null) + map.put("networkId", networkid); + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/PortForwarding.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/PortForwarding.java new file mode 100755 index 0000000..755202a --- /dev/null +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/PortForwarding.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.model; + +import java.util.Map; + +/** + * Author: Martin Skoviera + * Created on: 15-Oct-15 + * Description: POJO object for Load Balancer Policy (type 11) or Port Forwarding Rule Usage Data (type 12) + */ +public class PortForwarding extends UsageData { + @Override + protected void additionalMetadata(Map map) { + // nothing to do here + } +} diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/SnapshotUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Snapshot.java similarity index 86% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/SnapshotUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Snapshot.java index d34b0e8..b1caa02 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/SnapshotUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Snapshot.java @@ -24,7 +24,7 @@ * Created on: 15-Oct-15 * Description: POJO object for Snapshot Usage Data (type 9) */ -public class SnapshotUsageData extends UsageData { +public class Snapshot extends UsageData { // Size of the template, or ISO private Long size; @@ -39,4 +39,10 @@ public Long getSize() { public void setSize(Long size) { this.size = size; } + + @Override + protected void additionalMetadata(Map map) { + if (size != null) + map.put("size", size); + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Template.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Template.java new file mode 100755 index 0000000..e9c83bd --- /dev/null +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Template.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.model; + +import java.util.HashMap; +import java.util.Map; + +/** + * Author: Martin Skoviera + * Created on: 15-Oct-15 + * Description: POJO object for Template and ISO Usage Data, both type 7 (Template) and 8 (ISO) + */ +public class Template extends UsageData { + + // Size of the template, or ISO + private Long size; + private Long virtualsize; + + ///////////////////////////// + // Getters and Setters + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } + + public Long getVirtualsize() { + return virtualsize; + } + + public void setVirtualsize(Long virtualsize) { + this.virtualsize = virtualsize; + } + + @Override + protected void additionalMetadata(Map map) { + if (size != null) + map.put("size", size); + if (virtualsize != null) + map.put("virtualSize", virtualsize); + } +} diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/UsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/UsageData.java index 85990c2..02d4b3e 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/UsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/UsageData.java @@ -16,6 +16,7 @@ */ package ch.icclab.cyclops.model; +import com.google.gson.annotations.Expose; import org.joda.time.DateTime; import org.joda.time.chrono.ISOChronology; import org.joda.time.format.DateTimeFormat; @@ -33,12 +34,35 @@ */ public abstract class UsageData { - private String _class; private static String PREFIX = "CloudStack"; + @Expose + private String _class; + // Name of the account + @Expose private String account; + // Measurement's timestamp + @Expose + private Long time; + + // Measurement's usage + @Expose + private Object usage; + + // Measurement's unit + @Expose + private Object unit = "h"; + + // Metadata container + @Expose + private Map metadata; + + ////////////////////////////////////////////////////// + //==== everything below this goes into metadata ====// + ////////////////////////////////////////////////////// + // ID of the account private String accountid; @@ -51,10 +75,6 @@ public abstract class UsageData { // The range of time for which the usage is aggregated private String startdate; private String enddate; - private Long time; - - // String representation of the usage, including the units of usage (e.g. "Hrs" for VM running time) - private Object usage; // Virtual machine private String usageid; @@ -120,7 +140,34 @@ public static long getMilisForTime(String time) { public void prepareForSending() { usage = Double.parseDouble(rawusage); - time = getMilisForTime(startdate) / 1000; + time = getMilisForTime(startdate); + + // add global metadata + addToMetadata("accountId", accountid); + addToMetadata("accountId", accountid); + addToMetadata("domainId", domainid); + addToMetadata("usageId", usageid); + addToMetadata("description", description); + addToMetadata("zoneId", zoneid); + addToMetadata("project", project); + addToMetadata("projectId", projectid); + addToMetadata("offeringId", offeringid); + + // ask children to add their metadata + additionalMetadata(metadata); + } + + protected abstract void additionalMetadata(Map map); + + protected void addToMetadata(String str, Object obj) { + if (metadata == null) { + metadata = new HashMap<>(); + } + + // don't add null objects + if (str != null && !str.isEmpty() && obj != null) { + metadata.put(str, obj); + } } ///////////////////////////// @@ -243,4 +290,16 @@ public String getOfferingid() { public void setOfferingid(String offeringid) { this.offeringid = offeringid; } + + public Map getMetadata() { + return metadata; + } + + public Object getUnit() { + return unit; + } + + public void setUnit(Object unit) { + this.unit = unit; + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMAllocated.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMAllocated.java new file mode 100755 index 0000000..0ca3280 --- /dev/null +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMAllocated.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.model; + +import java.util.Map; + +/** + * Author: Martin Skoviera + * Created on: 15-Oct-15 + * Description: POJO object for Virtual Machine Usage Data, both type 1 (Running) and 2 (Allocated) + */ + +public class VMAllocated extends UsageData { + + // The ID of the virtual machine + private String virtualmachineid; + + // The name of the virtual machine + private String name; + + // The ID of the template or the ID of the parent template. The parent template value is present when the current template was created from a volume. + private String templateid; + + // Hypervisor + private String type; + + ///////////////////////////// + // Getters and Setters + + public String getVirtualmachineid() { + return virtualmachineid; + } + + public void setVirtualmachineid(String virtualmachineid) { + this.virtualmachineid = virtualmachineid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTemplateid() { + return templateid; + } + + public void setTemplateid(String templateid) { + this.templateid = templateid; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @Override + protected void additionalMetadata(Map map) { + if (type != null) + map.put("type", type); + if (name != null) + map.put("name", name); + if (templateid != null) + map.put("templateId", templateid); + if (virtualmachineid != null) + map.put("virtualmachineId", virtualmachineid); + } +} \ No newline at end of file diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMRunning.java similarity index 82% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMRunning.java index 61918d0..f68d2ac 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VMRunning.java @@ -25,7 +25,7 @@ * Description: POJO object for Virtual Machine Usage Data, both type 1 (Running) and 2 (Allocated) */ -public class VMUsageData extends UsageData { +public class VMRunning extends UsageData { // The ID of the virtual machine private String virtualmachineid; @@ -73,4 +73,16 @@ public String getType() { public void setType(String type) { this.type = type; } + + @Override + protected void additionalMetadata(Map map) { + if (type != null) + map.put("type", type); + if (name != null) + map.put("name", name); + if (templateid != null) + map.put("templateId", templateid); + if (virtualmachineid != null) + map.put("virtualmachineId", virtualmachineid); + } } \ No newline at end of file diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VPNUserUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VPNUser.java similarity index 86% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VPNUserUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VPNUser.java index eff812c..7454db1 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VPNUserUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VPNUser.java @@ -24,5 +24,9 @@ * Created on: 15-Oct-15 * Description: POJO object for VPN User Usage Data (type 14) */ -public class VPNUserUsageData extends UsageData { +public class VPNUser extends UsageData { + @Override + protected void additionalMetadata(Map map) { + // nothing to do here + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VolumeUsageData.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Volume.java similarity index 84% rename from collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VolumeUsageData.java rename to collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Volume.java index bfe9263..f423442 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/VolumeUsageData.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Volume.java @@ -24,7 +24,7 @@ * Created on: 15-Oct-15 * Description: POJO object for Volume Usage Data (type 6) */ -public class VolumeUsageData extends UsageData { +public class Volume extends UsageData { // The amount of storage allocated private Long size; @@ -50,4 +50,12 @@ public String getTemplateid() { public void setTemplateid(String templateid) { this.templateid = templateid; } + + @Override + protected void additionalMetadata(Map map) { + if (size != null) + map.put("size", size); + if (templateid != null) + map.put("templateId", templateid); + } } diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/publish/RabbitMQPublisher.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/publish/RabbitMQPublisher.java index 6716dfe..89cefe3 100644 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/publish/RabbitMQPublisher.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/publish/RabbitMQPublisher.java @@ -20,6 +20,7 @@ import ch.icclab.cyclops.util.PrettyGson; import ch.icclab.cyclops.util.loggers.DispatchLogger; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; @@ -132,7 +133,7 @@ protected void broadcast(Object content) { private void send(String exchange, Object content, String routing) { try { // first format - String message = new Gson().toJson(content); + String message = new GsonBuilder().excludeFieldsWithoutExposeAnnotation().create().toJson(content); // then send channel.basicPublish(exchange, routing, null, message.getBytes()); diff --git a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/util/GsonMapping.java b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/util/GsonMapping.java index c44759c..3f7dc5e 100755 --- a/collectors/cloudstack/src/main/java/ch/icclab/cyclops/util/GsonMapping.java +++ b/collectors/cloudstack/src/main/java/ch/icclab/cyclops/util/GsonMapping.java @@ -43,27 +43,31 @@ public Class getClassForElement(JsonElement readElement) { // now return corresponding classes switch (usagetype) { case 1: + return VMRunning.class; case 2: - return VMUsageData.class; + return VMAllocated.class; case 3: - return IPUsageData.class; + return IP.class; case 4: + return NetworkSent.class; case 5: - return NetworkUsageData.class; + return NetworkReceived.class; case 6: - return VolumeUsageData.class; + return Volume.class; case 7: + return Template.class; case 8: - return TemplateAndIsoUsageData.class; + return ISO.class; case 9: - return SnapshotUsageData.class; + return Snapshot.class; case 11: + return LoadBalancer.class; case 12: - return PolicyOrRuleUsageData.class; + return PortForwarding.class; case 13: - return NetworkOfferingUsageData.class; + return NetworkOffering.class; case 14: - return VPNUserUsageData.class; + return VPNUser.class; default: //returning null will trigger Gson's default behavior logger.error("Unknown class in received JSON - using default GSON behaviour"); diff --git a/collectors/openstack-ceilometer/config/openstack-ceilometer.conf b/collectors/openstack-ceilometer/config/openstack-ceilometer.conf index fc908ad..9390f63 100644 --- a/collectors/openstack-ceilometer/config/openstack-ceilometer.conf +++ b/collectors/openstack-ceilometer/config/openstack-ceilometer.conf @@ -2,13 +2,13 @@ OpenstackAccount=OpenStackAccount OpenstackPassword=OpenStackPassword KeystoneDomain=default -KeystoneTenant=admin +KeystoneTenant=services CeilometerUrl=http://OpenStackIP:8777/v2/ KeystoneUrl=http://OpenStackIP:5000/v2.0 MetersUrl=http://OpenStackIP:8777/v2/meters # HTTP and HTTPs ports for RE to be exposed at -ServerHTTPPort=8080 +ServerHTTPPort=8081 # Demo data parameter Demo=true @@ -23,7 +23,7 @@ PublisherBroadcastExchange=cyclops.ceilometer.collector.broadcast PublisherDispatchExchange=cyclops.ceilometer.collector.dispatch # Hibernate connection credentials -HibernateURL=jdbc:postgresql://localhost/openstack-ceilometer +HibernateURL=jdbc:postgresql://localhost/ceilometer HibernateUsername=ubuntu HibernatePassword=pass1234 HibernateDriver=org.postgresql.Driver diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/client/OpenStackUsageDownloader.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/client/OpenStackUsageDownloader.java index ec51349..67149f3 100644 --- a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/client/OpenStackUsageDownloader.java +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/client/OpenStackUsageDownloader.java @@ -17,9 +17,13 @@ package ch.icclab.cyclops.client; import ch.icclab.cyclops.load.model.Response; -import ch.icclab.cyclops.model.OpenStackCeilometerUsage; import ch.icclab.cyclops.model.OpenStackMeter; import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.model.ceilometerMeasurements.AbstractOpenStackCeilometerUsage; +import ch.icclab.cyclops.persistence.CumulativeMeterUsage; +import ch.icclab.cyclops.persistence.HibernateClient; +import ch.icclab.cyclops.util.CachedCumulativeUsage; +import ch.icclab.cyclops.util.Constant; import ch.icclab.cyclops.util.OpenStackClient; import com.google.gson.Gson; import org.apache.logging.log4j.LogManager; @@ -32,7 +36,9 @@ import org.restlet.util.Series; import java.io.IOException; +import java.lang.reflect.Constructor; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.concurrent.Callable; @@ -50,6 +56,10 @@ public class OpenStackUsageDownloader implements Callable { private OpenStackClient openStackClient; + private HashMap cumulativeMap; + + private HibernateClient hibernateClient; + // container for usage record response Response.UsageRecordResponse usageRecordResponse; @@ -59,8 +69,9 @@ public class OpenStackUsageDownloader implements Callable { * @param url that will be used for API call */ public OpenStackUsageDownloader(String url) { - openStackClient = new OpenStackClient(); + this.openStackClient = new OpenStackClient(); this.url = url; + this.cumulativeMap = CachedCumulativeUsage.getCachedCumulativeUsage(); } /** @@ -73,7 +84,7 @@ private String pullData() throws IOException { Series
headerValue; ClientResource clientResource = new ClientResource(url); - headerValue = new Series
(Header.class); + headerValue = new Series<>(Header.class); Request request = clientResource.getRequest(); String token; @@ -117,14 +128,30 @@ protected Integer getCount() { */ protected List performRequest(OpenStackMeter meter) { try { + hibernateClient = HibernateClient.getInstance(); + // first step is to pull data from OpenStack List genericData = new ArrayList<>(); - if(meter != null) { - String data = pullData(); - Gson gson = new Gson(); - OpenStackUsageData[] openStackUsageData = gson.fromJson(data, OpenStackUsageData[].class); + if (meter != null) { + String data = pullData(); + Gson gson = new Gson(); + OpenStackUsageData[] openStackUsageData = gson.fromJson(data, OpenStackUsageData[].class); for (int i = 0; i < openStackUsageData.length; i++) { - genericData.add(new OpenStackCeilometerUsage(openStackUsageData[i], meter)); + // Construct the pojo class and add it to the generic data list + Constructor constructor = Class.forName(Constant.METER_NAMES.get(meter.getName())).getConstructor(OpenStackUsageData.class, OpenStackMeter.class); + AbstractOpenStackCeilometerUsage openStackUsage = (AbstractOpenStackCeilometerUsage) constructor.newInstance(openStackUsageData[i], meter); + if (meter.getType().equals(Constant.CEILOMETER_CUMULATIVE_METER)) { + // Get the older usage (if exists) and compute the cumulative meter out of the two measurements + String usageKey = getUsageKey(openStackUsageData[i], meter); + CumulativeMeterUsage cumulativeMeterUsage = new CumulativeMeterUsage(openStackUsage, usageKey); + updateToExistingId(cumulativeMeterUsage); + openStackUsage.setUsage(getCumulativeUsage(openStackUsageData[i].getAvg(), usageKey)); + // Persist the cumulativeUsage in hibernate + hibernateClient.persistObject(cumulativeMeterUsage); + // Get the ID from the persisted object and store it in memory (HashMap) linked to the latest usage + cumulativeMap.put(usageKey, cumulativeMeterUsage); + } + genericData.add(openStackUsage); } } // return list of points @@ -136,8 +163,48 @@ protected List performRequest(OpenStackMeter meter) { } } + private void updateToExistingId(CumulativeMeterUsage cumulativeMeterUsage) { + hibernateClient = HibernateClient.getInstance(); + ArrayList cumulativeDataList = (ArrayList) hibernateClient.executeQuery("FROM CumulativeMeterUsage WHERE usageKey='" + cumulativeMeterUsage.getUsageKey() + "'"); + if(!cumulativeDataList.isEmpty()){ + for(CumulativeMeterUsage usageData : cumulativeDataList){ + cumulativeMeterUsage.setId(usageData.getId()); + } + } + } + @Override public Object call() throws Exception { return performRequest(new OpenStackMeter()); } + + private Double getCumulativeUsage(Double measurementUsage, String usageKey) { + Double result; + if (cumulativeMap.containsKey(usageKey)) + result = measurementUsage - cumulativeMap.get(usageKey).getUsageCounter(); + else { + hibernateClient = HibernateClient.getInstance(); + ArrayList cumulativeDataList = (ArrayList) hibernateClient.executeQuery("FROM CumulativeMeterUsage WHERE usageKey='" + usageKey + "'"); + if (!cumulativeDataList.isEmpty()) { + for (CumulativeMeterUsage data : cumulativeDataList) + cumulativeMap.put(data.getUsageKey(), data); + result = measurementUsage - cumulativeMap.get(usageKey).getUsageCounter(); + if(result<=0){ + result = measurementUsage; + } + } + else + //TODO: Review + result = 0.0; + } + return result; + } + + private String getUsageKey(OpenStackUsageData openStackUsageData, OpenStackMeter openStackMeter) { + String measurementName = openStackMeter.getName(); + String resourceId = (String) openStackUsageData.getGroupby().get("resource_id"); + String projectId = (String) openStackUsageData.getGroupby().get("project_id"); + String userId = (String) openStackUsageData.getGroupby().get("user_id"); + return measurementName.concat(resourceId).concat(projectId).concat(userId); + } } diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/load/model/Response.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/load/model/Response.java index 6234a0a..d1f5782 100755 --- a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/load/model/Response.java +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/load/model/Response.java @@ -16,7 +16,7 @@ */ package ch.icclab.cyclops.load.model; -import ch.icclab.cyclops.model.OpenStackCeilometerUsage; +import ch.icclab.cyclops.model.ceilometerMeasurements.AbstractOpenStackCeilometerUsage; import java.util.ArrayList; import java.util.List; @@ -66,10 +66,10 @@ public class UsageRecordResponse { // number of received responses private Integer count; // array of usage records - private List usagerecord; + private List usagerecord; // array for custom usage records - private List customusagerecord; + private List customusagerecord; /** * This method will go over every usage record and return DB Point representation @@ -105,11 +105,11 @@ public void setCount(Integer count) { this.count = count; } - public void setUsagerecord(List usagerecord) { + public void setUsagerecord(List usagerecord) { this.usagerecord = usagerecord; } - public void setCustomusagerecord(List customusagerecord) { + public void setCustomusagerecord(List customusagerecord) { this.customusagerecord = customusagerecord; } } diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/OpenStackCeilometerUsage.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/AbstractOpenStackCeilometerUsage.java similarity index 56% rename from collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/OpenStackCeilometerUsage.java rename to collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/AbstractOpenStackCeilometerUsage.java index c2235c5..ec0ee36 100644 --- a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/OpenStackCeilometerUsage.java +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/AbstractOpenStackCeilometerUsage.java @@ -1,5 +1,10 @@ -package ch.icclab.cyclops.model; +package ch.icclab.cyclops.model.ceilometerMeasurements; +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.persistence.CumulativeMeterUsage; +import ch.icclab.cyclops.persistence.HibernateClient; +import ch.icclab.cyclops.util.Constant; import org.joda.time.DateTime; import org.joda.time.chrono.ISOChronology; import org.joda.time.format.DateTimeFormat; @@ -27,36 +32,46 @@ * Created by Manu Perez on 30/05/16. */ -public class OpenStackCeilometerUsage { +public abstract class AbstractOpenStackCeilometerUsage { - private String _class; - - //Usage value - private Double usage; + private String _class = this.getClass().getSimpleName(); //Account name of the user private String account; + //Usage value + private Double usage; + //Measurement time as a timestamp private Long time; - //Meter name - private String meter_name; + // Dashboard graph representation + private String chartType; + + //Meter Unit + private String unit; //Meta hashmap - private HashMap metadata; + private HashMap metadata; - public OpenStackCeilometerUsage(OpenStackUsageData udr, OpenStackMeter meter) { + public AbstractOpenStackCeilometerUsage(OpenStackUsageData usageData, OpenStackMeter meter) { this.set_class(this.getClass().getSimpleName()); - - account = (String) udr.getGroupby().get("user_id"); - usage = udr.getAvg(); - meter_name = meter.getName(); - time = getMilisForTime(udr.getDuration_start()) / 1000; - metadata = fillMetaData(udr, meter); + +// if (meter.getType().equalsIgnoreCase(Constant.CEILOMETER_CUMULATIVE_METER)) +// usage = getCumulativeUsage(usageData); +// else + usage = usageData.getAvg(); + + account = (String) usageData.getGroupby().get("user_id"); + unit = usageData.getUnit(); + time = getMilisForTime(usageData.getDuration_start()) / 1000; + metadata = fillMetaData(usageData, meter); + unit = usageData.getUnit(); } + public abstract AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter); + /** * Will compute number of milliseconds from epoch to startDate * @@ -78,25 +93,28 @@ public static long getMilisForTime(String time) { return dt.getMillis(); } - private HashMap fillMetaData(OpenStackUsageData udr, OpenStackMeter meter){ - HashMap meta = new HashMap(); - meta.put("count", udr.getCount()); - meta.put("duration_end", udr.getDuration_end()); - meta.put("min", udr.getMin()); - meta.put("max", udr.getMax()); - meta.put("sum", udr.getSum()); - meta.put("period", udr.getPeriod()); - meta.put("period_end", udr.getPeriod_end()); - meta.put("duration", udr.getDuration()); - meta.put("period_start", udr.getPeriod_start()); - meta.put("unit", udr.getUnit()); + private HashMap fillMetaData(OpenStackUsageData udr, OpenStackMeter meter) { + HashMap meta = new HashMap(); meta.put("project_id", udr.getGroupby().get("project_id")); meta.put("resource_id", udr.getGroupby().get("resource_id")); - meta.put("type", meter.getType()); return meta; } + private Double getCumulativeUsage(OpenStackUsageData usageData){ + HibernateClient hibernateClient = HibernateClient.getInstance(); + CumulativeMeterUsage usage = (CumulativeMeterUsage) hibernateClient.getObject(CumulativeMeterUsage.class, 1l); + + Double last; + if (usage == null) { + last = 0.0; + } else { + last = usageData.getAvg() - usage.getUsageCounter(); + } + + return last; + } + public String get_class() { return _class; } @@ -136,4 +154,20 @@ public String getAccount() { public void setAccount(String account) { this.account = account; } + + public String getChartType() { + return chartType; + } + + public void setChartType(String chartType) { + this.chartType = chartType; + } + + public String getUnit() { + return unit; + } + + public void setUnit(String unit) { + this.unit = unit; + } } diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpu.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpu.java new file mode 100644 index 0000000..099da47 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpu.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerCpu extends AbstractOpenStackCeilometerUsage{ + + public OpenStackCeilometerCpu(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerCpu(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpuDelta.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpuDelta.java new file mode 100644 index 0000000..55ec183 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpuDelta.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerCpuDelta extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerCpuDelta(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerCpuDelta(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpuUtil.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpuUtil.java new file mode 100644 index 0000000..1920341 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerCpuUtil.java @@ -0,0 +1,39 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerCpuUtil extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerCpuUtil(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerCpuUtil(udr, meter); + } + + +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskAllocation.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskAllocation.java new file mode 100644 index 0000000..510841a --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskAllocation.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskAllocation extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskAllocation(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskAllocation(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskCapacity.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskCapacity.java new file mode 100644 index 0000000..df3fdaa --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskCapacity.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskCapacity extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskCapacity(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskCapacity(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceAllocation.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceAllocation.java new file mode 100644 index 0000000..ba5a6f4 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceAllocation.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceAllocation extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceAllocation(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceAllocation(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceCapacity.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceCapacity.java new file mode 100644 index 0000000..fc01d7d --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceCapacity.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceCapacity extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceCapacity(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceCapacity(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadBytes.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadBytes.java new file mode 100644 index 0000000..66c6b92 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadBytes.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceReadBytes extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceReadBytes(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceReadBytes(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadBytesRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadBytesRate.java new file mode 100644 index 0000000..649cc69 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadBytesRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceReadBytesRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceReadBytesRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceReadBytesRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadRequests.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadRequests.java new file mode 100644 index 0000000..8067dac --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadRequests.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceReadRequests extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceReadRequests(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceReadRequests(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadRequestsRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadRequestsRate.java new file mode 100644 index 0000000..3fca4ef --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceReadRequestsRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceReadRequestsRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceReadRequestsRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceReadRequestsRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceUsage.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceUsage.java new file mode 100644 index 0000000..83c66fb --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceUsage.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceUsage extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceUsage(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceUsage(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteBytes.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteBytes.java new file mode 100644 index 0000000..a725565 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteBytes.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceWriteBytes extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceWriteBytes(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceWriteBytes(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteBytesRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteBytesRate.java new file mode 100644 index 0000000..63886eb --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteBytesRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceWriteBytesRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceWriteBytesRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceWriteBytesRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteRequests.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteRequests.java new file mode 100644 index 0000000..3dd14a2 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteRequests.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceWriteRequests extends AbstractOpenStackCeilometerUsage{ + + public OpenStackCeilometerDiskDeviceWriteRequests(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceWriteRequests(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteRequestsRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteRequestsRate.java new file mode 100644 index 0000000..c9ff561 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskDeviceWriteRequestsRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskDeviceWriteRequestsRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskDeviceWriteRequestsRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskDeviceWriteRequestsRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadBytes.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadBytes.java new file mode 100644 index 0000000..382eb39 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadBytes.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskReadBytes extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskReadBytes(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskReadBytes(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadBytesRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadBytesRate.java new file mode 100644 index 0000000..3bc17fe --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadBytesRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskReadBytesRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskReadBytesRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskReadBytesRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadRequests.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadRequests.java new file mode 100644 index 0000000..3401447 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadRequests.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskReadRequests extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskReadRequests(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskReadRequests(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadRequestsRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadRequestsRate.java new file mode 100644 index 0000000..9733ac1 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskReadRequestsRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskReadRequestsRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskReadRequestsRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskReadRequestsRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskUsage.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskUsage.java new file mode 100644 index 0000000..363b0a8 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskUsage.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskUsage extends AbstractOpenStackCeilometerUsage{ + + public OpenStackCeilometerDiskUsage(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskUsage(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteBytes.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteBytes.java new file mode 100644 index 0000000..0de3816 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteBytes.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskWriteBytes extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskWriteBytes(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskWriteBytes(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteBytesRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteBytesRate.java new file mode 100644 index 0000000..13a0774 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteBytesRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskWriteBytesRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskWriteBytesRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskWriteBytesRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteRequests.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteRequests.java new file mode 100644 index 0000000..ca5e608 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteRequests.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskWriteRequests extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskWriteRequests(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskWriteRequests(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteRequestsRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteRequestsRate.java new file mode 100644 index 0000000..621027d --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerDiskWriteRequestsRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerDiskWriteRequestsRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerDiskWriteRequestsRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerDiskWriteRequestsRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImage.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImage.java new file mode 100644 index 0000000..a7773c4 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImage.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerImage extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerImage(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerImage(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageDownload.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageDownload.java new file mode 100644 index 0000000..2a8e95d --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageDownload.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerImageDownload extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerImageDownload(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerImageDownload(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageServe.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageServe.java new file mode 100644 index 0000000..2164ad4 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageServe.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerImageServe extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerImageServe(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerImageServe(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageSize.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageSize.java new file mode 100644 index 0000000..048653a --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerImageSize.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerImageSize extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerImageSize(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerImageSize(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerInstance.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerInstance.java new file mode 100644 index 0000000..f2dc524 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerInstance.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerInstance extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerInstance(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerInstance(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerIpFloating.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerIpFloating.java new file mode 100644 index 0000000..40c194a --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerIpFloating.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerIpFloating extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerIpFloating(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerIpFloating(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerMemoryResident.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerMemoryResident.java new file mode 100644 index 0000000..f323b83 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerMemoryResident.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerMemoryResident extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerMemoryResident(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerMemoryResident(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerMemoryUsage.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerMemoryUsage.java new file mode 100644 index 0000000..13f9e2d --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerMemoryUsage.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerMemoryUsage extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerMemoryUsage(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerMemoryUsage(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingBytes.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingBytes.java new file mode 100644 index 0000000..f4bc0b2 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingBytes.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerNetworkIncomingBytes extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerNetworkIncomingBytes(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerNetworkIncomingBytes(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingBytesRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingBytesRate.java new file mode 100644 index 0000000..f5090c7 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingBytesRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerNetworkIncomingBytesRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerNetworkIncomingBytesRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerNetworkIncomingBytesRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingPackets.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingPackets.java new file mode 100644 index 0000000..7261e34 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingPackets.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerNetworkIncomingPackets extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerNetworkIncomingPackets(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerNetworkIncomingPackets(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingPacketsRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingPacketsRate.java new file mode 100644 index 0000000..2f39d42 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkIncomingPacketsRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerNetworkIncomingPacketsRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerNetworkIncomingPacketsRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerNetworkIncomingPacketsRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingBytes.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingBytes.java new file mode 100644 index 0000000..21d96fd --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingBytes.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerNetworkOutgoingBytes extends AbstractOpenStackCeilometerUsage{ + + public OpenStackCeilometerNetworkOutgoingBytes(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerNetworkOutgoingBytes(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingBytesRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingBytesRate.java new file mode 100644 index 0000000..ef51c70 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingBytesRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerNetworkOutgoingBytesRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerNetworkOutgoingBytesRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerNetworkOutgoingBytesRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingPackets.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingPackets.java new file mode 100644 index 0000000..4c7fefc --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingPackets.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerNetworkOutgoingPackets extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerNetworkOutgoingPackets(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.NUMBER_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerNetworkOutgoingPackets(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingPacketsRate.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingPacketsRate.java new file mode 100644 index 0000000..63905d8 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerNetworkOutgoingPacketsRate.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerNetworkOutgoingPacketsRate extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerNetworkOutgoingPacketsRate(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerNetworkOutgoingPacketsRate(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjects.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjects.java new file mode 100644 index 0000000..ac6e320 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjects.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerStorageObjects extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerStorageObjects(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerStorageObjects(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjectsContainers.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjectsContainers.java new file mode 100644 index 0000000..74fdb22 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjectsContainers.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerStorageObjectsContainers extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerStorageObjectsContainers(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerStorageObjectsContainers(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjectsSize.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjectsSize.java new file mode 100644 index 0000000..77c3ffd --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/model/ceilometerMeasurements/OpenStackCeilometerStorageObjectsSize.java @@ -0,0 +1,37 @@ +package ch.icclab.cyclops.model.ceilometerMeasurements; + +import ch.icclab.cyclops.model.OpenStackMeter; +import ch.icclab.cyclops.model.OpenStackUsageData; +import ch.icclab.cyclops.util.Constant; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +public class OpenStackCeilometerStorageObjectsSize extends AbstractOpenStackCeilometerUsage { + + public OpenStackCeilometerStorageObjectsSize(OpenStackUsageData udr, OpenStackMeter meter) { + super(udr, meter); + this.setChartType(Constant.GAUGE_GRAPH); + } + + @Override + public AbstractOpenStackCeilometerUsage loadApplication(OpenStackUsageData udr, OpenStackMeter meter) { + return new OpenStackCeilometerStorageObjectsSize(udr, meter); + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/persistence/CumulativeMeterUsage.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/persistence/CumulativeMeterUsage.java new file mode 100644 index 0000000..6de23ee --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/persistence/CumulativeMeterUsage.java @@ -0,0 +1,72 @@ +package ch.icclab.cyclops.persistence; + +import ch.icclab.cyclops.model.ceilometerMeasurements.AbstractOpenStackCeilometerUsage; + +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 30/05/16. + */ + +@Entity +public class CumulativeMeterUsage { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private Double usageCounter; + + private String usageKey; + + public CumulativeMeterUsage() { + } + + public CumulativeMeterUsage(AbstractOpenStackCeilometerUsage openStackUsage, String usageKey) { + this.usageCounter = openStackUsage.getUsage(); + this.usageKey = usageKey; + } + + public Double getUsageCounter() { + return usageCounter; + } + + public void setUsageCounter(Double usageCounter) { + this.usageCounter = usageCounter; + } + + public String getUsageKey() { + return usageKey; + } + + public void setUsageKey(String usageKey) { + this.usageKey = usageKey; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/persistence/HibernateConfiguration.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/persistence/HibernateConfiguration.java index ce0c8eb..59e536a 100644 --- a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/persistence/HibernateConfiguration.java +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/persistence/HibernateConfiguration.java @@ -30,6 +30,7 @@ public static Configuration createConfiguration(HibernateCredentials credentials // add mandatory hibernate classes conf.addAnnotatedClass(LatestPullORM.class); + conf.addAnnotatedClass(CumulativeMeterUsage.class); // now set properties conf.setProperty("hibernate.connection.driver_class", credentials.getHibernateDriver()) diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/util/CachedCumulativeUsage.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/util/CachedCumulativeUsage.java new file mode 100644 index 0000000..8b180c3 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/util/CachedCumulativeUsage.java @@ -0,0 +1,36 @@ +package ch.icclab.cyclops.util; + +import ch.icclab.cyclops.persistence.CumulativeMeterUsage; + +import java.util.HashMap; + +/** + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *

+ * Created by Manu Perez on 07/09/16. + */ + +public class CachedCumulativeUsage { + public static HashMap cachedCumulativeUsage = new HashMap<>(); + + public static HashMap getCachedCumulativeUsage() { + return cachedCumulativeUsage; + } + + public static void setCachedCumulativeUsage(HashMap cachedCumulativeUsage) { + CachedCumulativeUsage.cachedCumulativeUsage = cachedCumulativeUsage; + } +} diff --git a/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/util/Constant.java b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/util/Constant.java new file mode 100644 index 0000000..d35c3f4 --- /dev/null +++ b/collectors/openstack-ceilometer/src/main/java/ch/icclab/cyclops/util/Constant.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.util; + +import java.util.HashMap; +import java.util.Map; + +public class Constant { + + // Ceilometer field name + public static final String CEILOMETER_METER_NAME = "name"; + + // Graph type constants + public static final String GAUGE_GRAPH = "gauge"; + public static final String HISTOGRAM_GRAPH = "histogram"; + public static final String NUMBER_GRAPH = "number"; + + // Ceilometer Meter names mapping map + public static final Map METER_NAMES = initializeMeternameMap(); + public static final String modelPath = "ch.icclab.cyclops.model.ceilometerMeasurements."; + public static final String CEILOMETER_CUMULATIVE_METER = "cumulative"; + + // Initialization of the metername map + private static Map initializeMeternameMap() { + HashMap map = new HashMap<>(); + map.put("cpu", modelPath + "Cpu"); + map.put("cpu.delta", modelPath + "CpuDelta"); + map.put("cpu_util", modelPath + "CpuUtil"); + map.put("disk.allocation", modelPath + "DiskAllocation"); + map.put("disk.capacity", modelPath + "DiskCapacity"); + map.put("disk.device.allocation", modelPath + "DiskDeviceAllocation"); + map.put("disk.device.capacity", modelPath + "DiskDeviceCapacity"); + map.put("disk.device.read.bytes", modelPath + "DiskDeviceReadBytes"); + map.put("disk.device.read.bytes.rate", modelPath + "DiskDeviceReadBytesRate"); + map.put("disk.device.read.requests", modelPath + "DiskDeviceReadRequest"); + map.put("disk.device.read.requests.rate", modelPath + "DiskDeviceReadRequestRate"); + map.put("disk.device.usage", modelPath + "DiskDeviceUsage"); + map.put("disk.device.write.bytes", modelPath + "DiskDeviceWriteBytes"); + map.put("disk.device.write.bytes.rate", modelPath + "DiskDeviceWriteBytesRate"); + map.put("disk.device.write.requests", modelPath + "DiskDeviceWriteRequests"); + map.put("disk.device.write.requests.rate", modelPath + "DiskDeviceWriteRequestsRate"); + map.put("disk.read.bytes", modelPath + "DiskReadBytes"); + map.put("disk.read.bytes.rate", modelPath + "DiskReadBytesRate"); + map.put("disk.read.requests", modelPath + "DiskReadRequests"); + map.put("disk.read.requests.rate", modelPath + "DiskReadRequestsRate"); + map.put("disk.usage", modelPath + "DiskUsage"); + map.put("disk.write.bytes", modelPath + "DiskWriteBytes"); + map.put("disk.write.bytes.rate", modelPath + "DiskWriteBytesRate"); + map.put("disk.write.requests", modelPath + "DiskWriteRequests"); + map.put("disk.write.requests.rate", modelPath + "DiskWriteRequestsRate"); + map.put("image", modelPath + "Image"); + map.put("image.download", modelPath + "ImageDownload"); + map.put("image.serve", modelPath + "ImageServe"); + map.put("image.size", modelPath + "ImageSize"); + map.put("instance", modelPath + "Instance"); + map.put("ip.floating", modelPath + "IpFloating"); + map.put("memory.resident", modelPath + "MemoryResident"); + map.put("memory.usage", modelPath + "MemoryUsage"); + map.put("network.incoming.bytes", modelPath + "NetworkIncomingBytes"); + map.put("network.incoming.bytes.rate", modelPath + "NetworkIncomingBytesRate"); + map.put("network.incoming.packets", modelPath + "NetworkIncomingPackets"); + map.put("network.incoming.packets.rate", modelPath + "NetworkIncomingPacketsRate"); + map.put("network.outgoing.bytes", modelPath + "NetworkOutgoingBytes"); + map.put("network.outgoing.bytes.rate", modelPath + "NetworkOutgoingBytesRate"); + map.put("network.outgoing.packets", modelPath + "NetworkOutgoingPackets"); + map.put("network.outgoing.packets.rate", modelPath + "NetworkOutgoingPacketsRate"); + map.put("storage.objects", modelPath + "StorageObjects"); + map.put("storage.objects.containers", modelPath + "StorageObjectsContainers"); + map.put("storage.objects.size", modelPath + "StorageObjectsSize"); + return map; + } + +} diff --git a/collectors/openstack-events/bin/openstack_event.jar b/collectors/openstack-events/bin/openstack_event.jar index c8e5425..c19a2f7 100644 Binary files a/collectors/openstack-events/bin/openstack_event.jar and b/collectors/openstack-events/bin/openstack_event.jar differ diff --git a/collectors/openstack-events/config/openstack_event.conf b/collectors/openstack-events/config/openstack_event.conf index f91ae4a..4fa0330 100644 --- a/collectors/openstack-events/config/openstack_event.conf +++ b/collectors/openstack-events/config/openstack_event.conf @@ -36,16 +36,14 @@ ConsumerVirtualHost=/ ConsumerDataQueue=cyclops.openstack.event.data # Some OpenStack settings -OpenstackCollectorEventStart=powering-on -OpenstackCollectorEventSpawn=spawning -OpenstackCollectorEventUnpause=unpausing -OpenstackCollectorEventResume=resuming -OpenstackCollectorEventStop=[powering-off] -OpenstackCollectorEventPause=pausing +OpenstackCollectorEventRun=running +OpenstackCollectorEventStop=poweredOff +OpenstackCollectorEventPause=paused OpenstackCollectorEventDelete=deleted OpenstackCollectorEventResize=resize_finish -OpenstackCollectorEventSuspend=suspending +OpenstackCollectorEventSuspend=suspended -OpenstackCollectorEventTable=events +OpenstackCollectorEventNovaTable=novaEvents +OpenstackCollectorEventNeutronTable=neutronEvents OpenstackCollectorScheduleTime=30000 diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/application/Main.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/application/Main.java index f20bf94..58fb008 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/application/Main.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/application/Main.java @@ -24,10 +24,11 @@ import ch.icclab.cyclops.persistence.HibernateClient; import ch.icclab.cyclops.persistence.HibernateConfiguration; import ch.icclab.cyclops.schedule.Scheduler; -import ch.icclab.cyclops.schedule.runner.OpenStackClient; import ch.icclab.cyclops.load.Settings; import ch.icclab.cyclops.load.model.PublisherCredentials; import ch.icclab.cyclops.publish.RabbitMQPublisher; +import ch.icclab.cyclops.schedule.runner.openstack.NeutronUDRRunner; +import ch.icclab.cyclops.schedule.runner.openstack.NovaUDRRunner; import ch.icclab.cyclops.timeseries.InfluxDBClient; import ch.icclab.cyclops.util.ShutDownListener; import org.apache.logging.log4j.LogManager; @@ -252,7 +253,8 @@ private static void checkAndStartServer(Component component) { // also start collection immediately Long time = new Long(Loader.getSettings().getOpenstackSettings().getOpenstackScheduleTime()); Scheduler scheduler = Scheduler.getInstance(); - scheduler.addRunner(new OpenStackClient(), 0, time, TimeUnit.MILLISECONDS); + scheduler.addRunner(new NovaUDRRunner(), 0, time, TimeUnit.MILLISECONDS); + scheduler.addRunner(new NeutronUDRRunner(), 0, time, TimeUnit.MILLISECONDS); scheduler.start(); // and finally start the server diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java index c961299..c9c1947 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java @@ -18,14 +18,18 @@ package ch.icclab.cyclops.consume.data; import ch.icclab.cyclops.consume.AbstractConsumer; -import ch.icclab.cyclops.consume.data.mapping.OpenstackEvent; -import ch.icclab.cyclops.consume.data.mapping.OsloEvent; -import ch.icclab.cyclops.consume.data.mapping.OsloEvent.OsloMessage.Args; -import ch.icclab.cyclops.consume.data.mapping.OsloEvent.OsloMessage.Args.ObjInst.Nova_objectData; +import ch.icclab.cyclops.consume.data.mapping.messages.NeutronEvent; +import ch.icclab.cyclops.consume.data.mapping.messages.OsloEvent; +import ch.icclab.cyclops.consume.data.mapping.messages.OsloEvent.OsloMessage.Args; +import ch.icclab.cyclops.consume.data.mapping.messages.OsloEvent.OsloMessage.Args.ObjInst.Nova_objectData; +import ch.icclab.cyclops.consume.data.mapping.openstack.OpenstackEvent; +import ch.icclab.cyclops.consume.data.mapping.openstack.events.OpenstackNeutronEvent; +import ch.icclab.cyclops.consume.data.mapping.openstack.events.OpenstackNovaEvent; import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.model.OpenstackSettings; import ch.icclab.cyclops.timeseries.InfluxDBClient; import com.google.gson.Gson; + import java.util.Arrays; import java.util.List; @@ -42,30 +46,30 @@ public class DataConsumer extends AbstractConsumer { @Override protected void consume(String content) { + OpenstackEvent data =null; try { - OpenstackEvent data = manageMessage(content); - - if (data.isValid()){ - influxDBClient.persistSinglePoint(data.getPoint()); - } + data = manageNovaMessage(content); } catch (Exception ignored) { - } + try{ + data = manageNeuntronMessage(content); + }catch (Exception ignored) { + } + + if (data != null) influxDBClient.persistSinglePoint(data.getPoint()); } - private OpenstackEvent manageMessage(String content) { + private OpenstackNovaEvent manageNovaMessage(String content) { Gson mapper = new Gson(); //list of nova methods related to nova InstanceUpTime List listOfActions = Arrays.asList( settings.getOpenstackCollectorEventDelete(), - settings.getOpenstackCollectorEventResize(), settings.getOpenstackCollectorEventSpawn(), - settings.getOpenstackCollectorEventStart(), settings.getOpenstackCollectorEventStop(), - settings.getOpenstackCollectorEventSuspend(), settings.getOpenstackCollectorEventResume(), - settings.getOpenstackCollectorEventUnpause(), settings.getOpenstackCollectorEventPause()); + settings.getOpenstackCollectorEventResize(), settings.getOpenstackCollectorEventStop(), + settings.getOpenstackCollectorEventSuspend(), settings.getOpenstackCollectorEventPause(), + settings.getOpenstackCollectorEventRun()); OsloEvent osloEvent = mapper.fromJson(content, OsloEvent.class); Args args = osloEvent.getOsloMessage().getArgs(); String method = ""; try{ - // method = args.getKwargs().getExpected_task_state().toString(); } catch (Exception ignored){ } @@ -74,15 +78,59 @@ private OpenstackEvent manageMessage(String content) { method = settings.getOpenstackCollectorEventDelete(); } } + String type = getType(method); String time = osloEvent.getOsloMessage().get_context_timestamp(); - if(listOfActions.contains(method)) { + if(listOfActions.contains(type)) { Nova_objectData novaData = args.getObjinst().getNova_objectData(); String instanceId = novaData.getUuid(); - String userName = osloEvent.getOsloMessage().get_context_user_name(); + String account = osloEvent.getOsloMessage().get_context_project_name(); Double memory = novaData.getMemory_mb(); Double vcpus = novaData.getVcpus(); - return new OpenstackEvent(userName,instanceId, method, memory, vcpus, time); + return new OpenstackNovaEvent(account,instanceId, type, memory, vcpus, time); + } + return null; + } + + private OpenstackNeutronEvent manageNeuntronMessage(String content) { + Gson mapper = new Gson(); + List listOfActions = Arrays.asList( settings.getOpenstackCollectorEventDelete(), settings.getOpenstackCollectorEventRun()); + NeutronEvent neutronEvent = mapper.fromJson(content, NeutronEvent.class); + String id; + if (neutronEvent.getPayload().getFloatingip_id() != null) { + id = neutronEvent.getPayload().getFloatingip_id(); + } else { + id = neutronEvent.getPayload().getFloatingip().getId(); + } + String type = getType(neutronEvent.getEvent_type()); + if (listOfActions.contains(type)){ + return new OpenstackNeutronEvent(neutronEvent.get_context_tenant_name(), id, type, neutronEvent.get_context_timestamp()); } + return null; } + + private String getType(String method){ + List listOfRunningActions = Arrays.asList("spawning", "powering-on", "unpausing", "resuming", "floatingip.create.end"); + String paused = "pausing"; + String poweredOff="[powering-off]"; + String suspend = "suspending"; + String deleted = "floatingip.delete.end"; + + if (listOfRunningActions.contains(method)){ + return settings.getOpenstackCollectorEventRun(); + } + if (method.equals(paused)){ + return settings.getOpenstackCollectorEventPause(); + } + if (method.equals(poweredOff)){ + return settings.getOpenstackCollectorEventStop(); + } + if (method.equals(suspend)){ + return settings.getOpenstackCollectorEventSuspend(); + } + if (method.equals(deleted)){ + return settings.getOpenstackCollectorEventDelete(); + } + return method; + } } diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/messages/NeutronEvent.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/messages/NeutronEvent.java new file mode 100644 index 0000000..6556bb7 --- /dev/null +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/messages/NeutronEvent.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.consume.data.mapping.messages; + +/** + * Author: Oleksii Serhiienko + * Updated on: 26-Aug-16 + * Description: This class holds the Neutron event data + */ +public class NeutronEvent { + public String getEvent_type() { + return event_type; + } + + public void setEvent_type(String event_type) { + this.event_type = event_type; + } + + public String get_context_timestamp() { + return _context_timestamp; + } + + public void set_context_timestamp(String _context_timestamp) { + this._context_timestamp = _context_timestamp; + } + + public String get_context_tenant_name() { + return _context_tenant_name; + } + + public void set_context_tenant_name(String _context_tenant_name) { + this._context_tenant_name = _context_tenant_name; + } + + public Payload getPayload() { + return payload; + } + + public void setPayload(Payload payload) { + this.payload = payload; + } + + public String event_type; + public String _context_timestamp; + public String _context_tenant_name; + public Payload payload; + public class Payload { + public String getFloatingip_id() { + return floatingip_id; + } + + public void setFloatingip_id(String floatingip_id) { + this.floatingip_id = floatingip_id; + } + + public String floatingip_id; + + public FloatingIP getFloatingip() { + return floatingip; + } + + public void setFloatingip(FloatingIP floatingip) { + this.floatingip = floatingip; + } + + public FloatingIP floatingip; + public class FloatingIP { + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + public String id; + } + } +} diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OsloEvent.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/messages/OsloEvent.java similarity index 93% rename from collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OsloEvent.java rename to collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/messages/OsloEvent.java index 5df3e6e..5c4edc3 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OsloEvent.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/messages/OsloEvent.java @@ -14,8 +14,7 @@ * License for the specific language governing permissions and limitations * under the License. */ - -package ch.icclab.cyclops.consume.data.mapping; +package ch.icclab.cyclops.consume.data.mapping.messages; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; @@ -37,11 +36,11 @@ public Args getArgs() { } Args args; - String _context_user_name; + String _context_project_name; String _context_timestamp; - public String get_context_user_name() { - return _context_user_name; + public String get_context_project_name() { + return _context_project_name; } public String get_context_timestamp() { return _context_timestamp; } diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OpenstackEvent.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/OpenstackEvent.java similarity index 51% rename from collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OpenstackEvent.java rename to collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/OpenstackEvent.java index 6d555c6..f8d724b 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OpenstackEvent.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/OpenstackEvent.java @@ -14,49 +14,33 @@ * License for the specific language governing permissions and limitations * under the License. */ -package ch.icclab.cyclops.consume.data.mapping; +package ch.icclab.cyclops.consume.data.mapping.openstack; -import ch.icclab.cyclops.load.Loader; -import ch.icclab.cyclops.util.Time; import org.influxdb.dto.Point; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import static java.util.concurrent.TimeUnit.MILLISECONDS; /** - * Author: Oleksii - * Date: 01/04/2016 - * Description: This class holds the OpenstackEvent data + * Author: Oleksii Serhiienko + * Updated on: 26-Aug-16 + * Description: This abstract class holds the OpenstackEvent data */ -public class OpenstackEvent { +public abstract class OpenstackEvent { - public OpenstackEvent(String userName,String instanceId, String action, Double memory, Double vcpus, String time){ - this.userName = userName; - this.instanceId = instanceId; - this.action = action; - this.memory = memory; - this.vcpus = vcpus; - this.time = time; - } - - private String userName; - - public String getUserName() { - return userName; + public String getAccount() { + return account; } - public void setUserName(String userName) { - this.userName = userName; + public void setAccount(String account) { + this.account = account; } - private String instanceId; - public String getInstanceId() { return instanceId; } @@ -65,41 +49,38 @@ public void setInstanceId(String instanceId) { this.instanceId = instanceId; } - private String action; - - public String getAction() { - return action; + public String getType() { + return type; } - public void setAction(String action) { - this.action = action; + public void setType(String type) { + this.type = type; } - private String time; - - public String getTime() { return time; } - - public void setTime(String time) { this.time = time; } - - private Double memory; - - private Double vcpus; - - public void setMemory(Double memory) {this.memory = memory;} + public String getTime() { + return time; + } - public void setVcpus(Double vcpus) {this.vcpus = vcpus;} + public void setTime(String time) { + this.time = time; + } public boolean isValid() { - return action != null; + return type != null; } + public String account; + public String instanceId; + public String type; + public String time; + /** * This public method will access data and create db Point * * @return db point */ public Point getPoint() { - DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS").withZoneUTC(); + DateTimeFormatter formatter = DateTimeFormat.forPattern(getDateFormat()).withZoneUTC(); DateTime dt = formatter.parseDateTime(time); Map fields = getFields(); removeNullValues(fields); @@ -119,27 +100,10 @@ private void removeNullValues(Map map) { map.values().removeAll(Collections.singleton(null)); } - /** - * @return table - */ - private String getTableName() { - return Loader.getSettings().getOpenstackSettings().getOpenstackEventTable(); - } + protected abstract String getTableName(); - /** - * Get fields for point generation - * - * @return hashmap - */ - private Map getFields() { - Map map = new HashMap(); - map.put("instanceId", instanceId); - map.put("clientId", userName); - map.put("status", action); - map.put("memory", memory.toString()); - map.put("cpu", vcpus.toString()); - - return map; - } + protected abstract Map getFields(); + + protected abstract String getDateFormat(); } diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/events/OpenstackNeutronEvent.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/events/OpenstackNeutronEvent.java new file mode 100644 index 0000000..96addea --- /dev/null +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/events/OpenstackNeutronEvent.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.consume.data.mapping.openstack.events; + +import ch.icclab.cyclops.consume.data.mapping.openstack.OpenstackEvent; +import ch.icclab.cyclops.load.Loader; + +import java.util.HashMap; +import java.util.Map; + +/** + * Author: Oleksii Serhiienko + * Updated on: 26-Aug-16 + * Description: This class holds the OpenstackNeutronEvent data + */ +public class OpenstackNeutronEvent extends OpenstackEvent { + public OpenstackNeutronEvent(String account, String instanceId, String type, String time){ + this.account = account; + this.instanceId = instanceId; + this.type = type; + this.time = time; + } + + /** + * @return table + */ + public String getTableName() { + return Loader.getSettings().getOpenstackSettings().getOpenstackEventNeutronTable(); + } + + /** + * Get fields for point generation + * + * @return hashmap + */ + public Map getFields() { + Map map = new HashMap(); + map.put("instanceId", instanceId); + map.put("account", account); + map.put("type", type); + + return map; + } + + public String getDateFormat(){ + return "yyyy-MM-dd HH:mm:ss.SSSSSS"; + } +} diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/events/OpenstackNovaEvent.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/events/OpenstackNovaEvent.java new file mode 100644 index 0000000..3d9ff21 --- /dev/null +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/openstack/events/OpenstackNovaEvent.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.consume.data.mapping.openstack.events; + +import ch.icclab.cyclops.consume.data.mapping.openstack.OpenstackEvent; +import ch.icclab.cyclops.load.Loader; + +import java.util.HashMap; +import java.util.Map; + +/** + * Author: Oleksii + * Date: 01/04/2016 + * Description: This class holds the OpenstackNovaEvent data + */ +public class OpenstackNovaEvent extends OpenstackEvent{ + + public OpenstackNovaEvent(String account, String instanceId, String type, Double memory, Double vcpus, String time){ + this.account = account; + this.instanceId = instanceId; + this.type = type; + this.memory = memory; + this.vcpus = vcpus; + this.time = time; + } + + private Double memory; + + private Double vcpus; + + public void setMemory(Double memory) {this.memory = memory;} + + public void setVcpus(Double vcpus) {this.vcpus = vcpus;} + + public Double getMemory() { return memory; } + + public Double getVcpus() { return vcpus; } + + /** + * @return table + */ + public String getTableName() { + return Loader.getSettings().getOpenstackSettings().getOpenstackEventNovaTable(); + } + + /** + * Get fields for point generation + * + * @return hashmap + */ + public Map getFields() { + Map map = new HashMap(); + map.put("instanceId", instanceId); + map.put("account", account); + map.put("type", type); + map.put("memory", memory.toString()); + map.put("cpu", vcpus.toString()); + + return map; + } + public String getDateFormat(){ + return "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"; + } + + +} diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackIpTimeUDR.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackIpTimeUDR.java new file mode 100644 index 0000000..1e73b5a --- /dev/null +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackIpTimeUDR.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.consume.data.mapping.udr; + +/** + * Author: Oleksii Serhiienko + * Updated on: 26-Aug-16 + * Description: This class holds the OpenStackIpTimeUDR response + */ + +public class OpenStackIpTimeUDR extends OpenStackUDR { + public OpenStackIpTimeUDR(Long time, String account, String instanceId, String state, Double usage){ + this.account = account; + this.usage = usage; + this.time = time; + this.metadata = new NeutronMetadata(instanceId, state); + } + + private String unit = "sec"; + private String charType = "Infogram"; + private String _class = getClass().getSimpleName(); + private NeutronMetadata metadata; + + private class NeutronMetadata { + private String instanceId; + private String state; + NeutronMetadata(String instanceId, String state){ + this.instanceId = instanceId; + this.state = state; + } + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBillCDRMapping.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackUDR.java similarity index 63% rename from core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBillCDRMapping.java rename to collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackUDR.java index a8d167c..8854083 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBillCDRMapping.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackUDR.java @@ -1,4 +1,3 @@ -package ch.icclab.cyclops.consume.command.model.localbill; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -15,17 +14,26 @@ * License for the specific language governing permissions and limitations * under the License. */ +package ch.icclab.cyclops.consume.data.mapping.udr; /** - * Author: Skoviera - * Created: 28/06/16 - * Description: Mapping for local CDR bill generation + * Author: Oleksii + * Date: 26/08/2016 + * Description: This class holds the OpenStackUDR response */ -public class LocalBillCDRMapping { - private String account; - private Double charge; +public class OpenStackUDR { - public LocalBillCDRMapping() {} + protected Long time; + protected String account; + protected Double usage; + + public Long getTime() { + return time; + } + + public void setTime(Long time) { + this.time = time; + } public String getAccount() { return account; @@ -35,11 +43,11 @@ public void setAccount(String account) { this.account = account; } - public Double getCharge() { - return charge; + public Double getUsage() { + return usage; } - public void setCharge(Double charge) { - this.charge = charge; + public void setUsage(Double usage) { + this.usage = usage; } } diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackUpTimeUDR.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackUpTimeUDR.java new file mode 100644 index 0000000..45d810f --- /dev/null +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/udr/OpenStackUpTimeUDR.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.consume.data.mapping.udr; + +/** + * Author: Oleksii + * Date: 01/06/2016 + * Description: This class holds the OpenStackUpTimeUDR response + */ + +public class OpenStackUpTimeUDR extends OpenStackUDR{ + + public OpenStackUpTimeUDR(Long time, String account, String instanceId, String state, Double usage, Double cpu, Double memory){ + this.account = account; + this.usage = usage; + this.time = time; + this.metadata = new NovaMetadata(instanceId, state, cpu, memory); + } + + private String unit = "sec"; + private String charType = "Infogram"; + private String _class = getClass().getSimpleName(); + private NovaMetadata metadata; + + private class NovaMetadata { + private Double cpu; + private Double memory; + private String instanceId; + private String state; + NovaMetadata(String instanceId, String state, Double cpu, Double memory){ + this.instanceId = instanceId; + this.state = state; + this.cpu = cpu; + this.memory = memory; + } + } + +} diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/Loader.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/Loader.java index 750be45..e4a8fd7 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/Loader.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/Loader.java @@ -19,8 +19,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.restlet.Context; -import javax.servlet.ServletContext; + import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/Settings.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/Settings.java index 5c1c310..7c7de64 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/Settings.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/Settings.java @@ -60,16 +60,14 @@ private OpenstackSettings loadOpenstackSettings() { openstackSettings.setOpenstackFirstImport(properties.getProperty("OpenstackFirstImportDate")); - openstackSettings.setOpenstackCollectorEventStart(properties.getProperty("OpenstackCollectorEventStart")); - openstackSettings.setOpenstackCollectorEventSpawn(properties.getProperty("OpenstackCollectorEventSpawn")); - openstackSettings.setOpenstackCollectorEventUnpause(properties.getProperty("OpenstackCollectorEventUnpause")); - openstackSettings.setOpenstackCollectorEventResume( properties.getProperty("OpenstackCollectorEventResume")); + openstackSettings.setOpenstackCollectorEventRun(properties.getProperty("OpenstackCollectorEventRun")); openstackSettings.setOpenstackCollectorEventPause( properties.getProperty("OpenstackCollectorEventPause")); openstackSettings.setOpenstackCollectorEventStop( properties.getProperty("OpenstackCollectorEventStop")); openstackSettings.setOpenstackCollectorEventDelete( properties.getProperty("OpenstackCollectorEventDelete")); openstackSettings.setOpenstackCollectorEventResize( properties.getProperty("OpenstackCollectorEventResize")); openstackSettings.setOpenstackCollectorEventSuspend( properties.getProperty("OpenstackCollectorEventSuspend")); - openstackSettings.setOpenstackEventTable(properties.getProperty("OpenstackCollectorEventTable")); + openstackSettings.setOpenstackEventNeutronTable(properties.getProperty("OpenstackCollectorEventNeutronTable")); + openstackSettings.setOpenstackEventNovaTable(properties.getProperty("OpenstackCollectorEventNovaTable")); openstackSettings.setOpenstackScheduleTime(properties.getProperty("OpenstackCollectorScheduleTime")); return openstackSettings; diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/model/HibernateCredentials.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/model/HibernateCredentials.java index e8da7b7..182aec4 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/model/HibernateCredentials.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/model/HibernateCredentials.java @@ -31,9 +31,7 @@ public class HibernateCredentials { public String getHibernateURL() { return hibernateURL; } - public void setHibernateURL(String hibernateURL) { - this.hibernateURL = hibernateURL; - } + public void setHibernateURL(String hibernateURL) { this.hibernateURL = hibernateURL; } public String getHibernateUsername() { return hibernateUsername; diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/model/OpenstackSettings.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/model/OpenstackSettings.java index a330cd9..21331e5 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/model/OpenstackSettings.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/load/model/OpenstackSettings.java @@ -24,38 +24,23 @@ */ public class OpenstackSettings { - private String OpenstackCollectorEventSpawn; - private String OpenstackCollectorEventUnpause; - private String OpenstackCollectorEventResume; - private String OpenstackCollectorEventStart; + private String OpenstackCollectorEventRun; private String OpenstackCollectorEventStop; private String OpenstackCollectorEventPause; private String OpenstackCollectorEventDelete; private String OpenstackCollectorEventResize; private String OpenstackCollectorEventSuspend; - private String OpenstackFirstImport; - private String OpenstackEventTable; - - private String OpenstackScheduleTime; - + private String OpenstackEventNovaTable; - public void setOpenstackCollectorEventSpawn(String openstackCollectorEventSpawn) { - OpenstackCollectorEventSpawn = openstackCollectorEventSpawn; - } - - public void setOpenstackCollectorEventUnpause(String openstackCollectorEventUnpause) { - OpenstackCollectorEventUnpause = openstackCollectorEventUnpause; - } + private String OpenstackEventNeutronTable; - public void setOpenstackCollectorEventResume(String openstackCollectorEventResume) { - OpenstackCollectorEventResume = openstackCollectorEventResume; - } + private String OpenstackScheduleTime; - public void setOpenstackCollectorEventStart(String openstackCollectorEventStart) { - OpenstackCollectorEventStart = openstackCollectorEventStart; + public void setOpenstackCollectorEventRun(String openstackCollectorEventRun) { + OpenstackCollectorEventRun = openstackCollectorEventRun; } public void setOpenstackCollectorEventStop(String openstackCollectorEventStop) { @@ -83,25 +68,22 @@ public void setOpenstackFirstImport(String openstackFirstImport) { } - public void setOpenstackEventTable(String openstackEventTable) { - OpenstackEventTable = openstackEventTable; - } - public void setOpenstackScheduleTime(String openstackScheduleTime) { OpenstackScheduleTime = openstackScheduleTime; } - public String getOpenstackCollectorEventSpawn() { - return OpenstackCollectorEventSpawn; - } - public String getOpenstackCollectorEventUnpause() { - return OpenstackCollectorEventUnpause; + public void setOpenstackEventNeutronTable(String openstackEventNeutronTable) { + OpenstackEventNeutronTable = openstackEventNeutronTable; } - public String getOpenstackCollectorEventResume() { - return OpenstackCollectorEventResume; + + public void setOpenstackEventNovaTable(String openstackEventNovaTable) { + OpenstackEventNovaTable = openstackEventNovaTable; } - public String getOpenstackCollectorEventStart() { - return OpenstackCollectorEventStart; + + public String getOpenstackEventNeutronTable() { return OpenstackEventNeutronTable; } + public String getOpenstackEventNovaTable() { return OpenstackEventNovaTable; } + public String getOpenstackCollectorEventRun() { + return OpenstackCollectorEventRun; } public String getOpenstackCollectorEventStop() { return OpenstackCollectorEventStop; } public String getOpenstackCollectorEventPause() { return OpenstackCollectorEventPause; } @@ -113,10 +95,6 @@ public String getOpenstackFirstImport() { return OpenstackFirstImport; } - public String getOpenstackEventTable() { - return OpenstackEventTable; - } - public String getOpenstackScheduleTime() { return OpenstackScheduleTime; } diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/HibernateConfiguration.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/HibernateConfiguration.java index ce0c8eb..5987012 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/HibernateConfiguration.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/HibernateConfiguration.java @@ -17,6 +17,8 @@ */ import ch.icclab.cyclops.load.model.HibernateCredentials; +import ch.icclab.cyclops.persistence.pulls.LatestPullNeutron; +import ch.icclab.cyclops.persistence.pulls.LatestPullNova; import org.hibernate.cfg.Configuration; /** @@ -29,7 +31,8 @@ public static Configuration createConfiguration(HibernateCredentials credentials Configuration conf = new Configuration(); // add mandatory hibernate classes - conf.addAnnotatedClass(LatestPullORM.class); + conf.addAnnotatedClass(LatestPullNova.class); + conf.addAnnotatedClass(LatestPullNeutron.class); // now set properties conf.setProperty("hibernate.connection.driver_class", credentials.getHibernateDriver()) diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/pulls/LatestPullNeutron.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/pulls/LatestPullNeutron.java new file mode 100644 index 0000000..706bf01 --- /dev/null +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/pulls/LatestPullNeutron.java @@ -0,0 +1,56 @@ + +/* + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.persistence.pulls; + +import javax.persistence.*; + +/** + * Author: Oleksii Serhiienko + * Updated on: 26-Aug-16 + * Description: + */ +@Entity +public class LatestPullNeutron { + @Id + @GeneratedValue(strategy= GenerationType.IDENTITY) + private Long id; + + public LatestPullNeutron() {} + + public LatestPullNeutron(Long timeStamp) { + this.timeStamp = timeStamp; + } + + private Long timeStamp; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getTimeStamp() { + return timeStamp; + } + + public void setTimeStamp(Long timeStamp) { + this.timeStamp = timeStamp; + } +} diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/LatestPullORM.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/pulls/LatestPullNova.java similarity index 76% rename from collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/LatestPullORM.java rename to collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/pulls/LatestPullNova.java index c3c902d..c32c977 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/LatestPullORM.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/persistence/pulls/LatestPullNova.java @@ -1,6 +1,5 @@ -package ch.icclab.cyclops.persistence; /* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,24 +14,24 @@ * License for the specific language governing permissions and limitations * under the License. */ +package ch.icclab.cyclops.persistence.pulls; import javax.persistence.*; /** - * Author: Skoviera - * Created: 23/05/16 + * Author: Oleksii Serhiienko + * Updated on: 26-Aug-16 * Description: */ @Entity -public class LatestPullORM { +public class LatestPullNova{ @Id - @GeneratedValue(strategy=GenerationType.IDENTITY) + @GeneratedValue(strategy= GenerationType.IDENTITY) private Long id; - public LatestPullORM() { - } + public LatestPullNova() {} - public LatestPullORM(Long timeStamp) { + public LatestPullNova(Long timeStamp) { this.timeStamp = timeStamp; } @@ -53,4 +52,5 @@ public Long getTimeStamp() { public void setTimeStamp(Long timeStamp) { this.timeStamp = timeStamp; } + } diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/Scheduler.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/Scheduler.java index 3104d8e..6b4d76d 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/Scheduler.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/Scheduler.java @@ -18,14 +18,15 @@ package ch.icclab.cyclops.schedule; import ch.icclab.cyclops.schedule.runner.AbstractRunner; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; /** * Author: Martin Skoviera diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/OpenStackClient.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/OpenStackClient.java index 82931e6..c326add 100755 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/OpenStackClient.java +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/OpenStackClient.java @@ -16,12 +16,11 @@ */ package ch.icclab.cyclops.schedule.runner; -import ch.icclab.cyclops.consume.data.mapping.OpenStackEventUDR; +import ch.icclab.cyclops.consume.data.mapping.udr.OpenStackUDR; import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.model.OpenstackSettings; import ch.icclab.cyclops.load.model.PublisherCredentials; import ch.icclab.cyclops.persistence.HibernateClient; -import ch.icclab.cyclops.persistence.LatestPullORM; import ch.icclab.cyclops.publish.Messenger; import ch.icclab.cyclops.timeseries.InfluxDBClient; import ch.icclab.cyclops.timeseries.QueryBuilder; @@ -29,7 +28,9 @@ import ch.icclab.cyclops.util.loggers.SchedulerLogger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.joda.time.*; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + import java.util.*; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -41,11 +42,11 @@ * Updated on: 01-July-16 * Description: Client class for transforming events to UDR Records for Openstack events */ -public class OpenStackClient extends AbstractRunner { +public abstract class OpenStackClient extends AbstractRunner { final static Logger logger = LogManager.getLogger(OpenStackClient.class.getName()); //link to hibernate - private HibernateClient hibernateClient; + protected HibernateClient hibernateClient; //link to influxDB client private static InfluxDBClient influxDBClient; //Openstack settings @@ -64,7 +65,7 @@ public OpenStackClient() { influxDBClient = InfluxDBClient.getInstance(); settings = Loader.getSettings().getOpenstackSettings(); publisherCredentials = Loader.getSettings().getPublisherCredentials(); - dbName = Loader.getSettings().getOpenstackSettings().getOpenstackEventTable(); + dbName = getDbName(); messenger = Messenger.getInstance(); } @@ -77,6 +78,14 @@ public void run() { transformEventsToUDRs(); } + public abstract String getDbName(); + + public abstract OpenStackUDR generateValue(Long eventTime, Long eventLastTime, Map lastEventInScope, String instanceId); + + public abstract void updateLatestPull(Long time); + + public abstract DateTime getLatestPull(); + /* * Transform Openstack Events to UDR records */ @@ -87,76 +96,54 @@ private void transformEventsToUDRs() { List data = influxDBClient.executeQuery(parameterQuery); SchedulerLogger.log("Influxdb data is successfully fetched."); SchedulerLogger.log("Making map of client and instance IDs..."); - HashMap> clientInstanceMap = getInstanceIdsPerClientId(data); + ArrayList instanceList = getListOfInstances(data); SchedulerLogger.log("Map of client and instance IDs is done"); - createUDRRecords(clientInstanceMap); + createUDRRecords(instanceList); } - /** * This method takes the POJOobject that contains all events, extracts all clientIDs - * and maps instanceIds to them which are saved to a HashMap. * * @param data * @return */ - private HashMap> getInstanceIdsPerClientId(List data) { - HashMap> map = new HashMap>(); + private ArrayList getListOfInstances(List data) { + ArrayList listOfInstances = new ArrayList<>(); for (Map obj : data) { - String clienId = obj.get("clientId").toString(); - String instanceId = obj.get("instanceId").toString(); - if (!map.containsKey(clienId)) { - map.put(clienId, new ArrayList<>()); - } - if (!map.get(clienId).contains(instanceId)){ - map.get(clienId).add(instanceId); + String instaceId = obj.get("instanceId").toString(); + if (!(listOfInstances.contains(instaceId))){ + listOfInstances.add(instaceId); } } - return map; + return listOfInstances; } - private void createUDRRecords(HashMap> clientInstanceMap) { + private void createUDRRecords(ArrayList listOfInstances) { SchedulerLogger.log("UDR creation process is started... "); - Iterator it = clientInstanceMap.entrySet().iterator(); DateInterval dates = new DateInterval(whenWasLastPull()); SchedulerLogger.log("The last pull was " + dates.fromDate + " " + dates.toDate); Long time = Time.getMilisForTime(dates.getToDate()); SchedulerLogger.log("Current timestamp is " + time); - ArrayList eventList = new ArrayList(); - while (it.hasNext()) { - Map.Entry pair = (Map.Entry) it.next(); - String clientId = pair.getKey().toString(); - ArrayList instanceIds = (ArrayList) pair.getValue(); - for (String instanceId : instanceIds) { + ArrayList eventList = new ArrayList<>(); + for (String instanceId : listOfInstances) { try { - ArrayList udr = generateUDR(clientId, instanceId, dates); + ArrayList udr = generateUDR(instanceId, dates); if (udr !=null){ eventList.addAll(udr); } } catch (Exception e) { SchedulerLogger.log("Couldn't generate UDR " +e); } - - } - it.remove(); } if (publisherCredentials.getPublisherByDefaultDispatchInsteadOfBroadcast()) { - messenger.publish(eventList, OpenStackEventUDR.class.getSimpleName()); + messenger.publish(eventList, OpenStackUDR.class.getSimpleName()); } else { messenger.broadcast(eventList); } SchedulerLogger.log("All udr are published "); + updateLatestPull(time); // update time stamp - LatestPullORM pull = (LatestPullORM) hibernateClient.getObject(LatestPullORM.class, 1l); - if (pull == null) { - pull = new LatestPullORM(time); - } else { - pull.setTimeStamp(time); - } - SchedulerLogger.log("The last pull set to "+pull.getTimeStamp().toString()); - hibernateClient.persistObject(pull); - } /** @@ -181,7 +168,7 @@ String getToDate() { } } - private ArrayList generateUDR(String clientId, String instanceId, DateInterval dates) { + private ArrayList generateUDR(String instanceId, DateInterval dates) { ArrayList generatedEvents = new ArrayList<>(); // generate first event @@ -190,8 +177,8 @@ private ArrayList generateUDR(String clientId, String instanc Boolean isItExist = true; try { - Map lastEvent = getEventBeforeTime(fromMills, clientId, instanceId); - if (lastEvent.get("status").equals(settings.getOpenstackCollectorEventDelete())){ + Map lastEvent = getEventBeforeTime(fromMills, instanceId); + if (lastEvent.get("type").equals(settings.getOpenstackCollectorEventDelete())){ isItExist = false; } else { @@ -203,20 +190,18 @@ private ArrayList generateUDR(String clientId, String instanc if (isItExist) { //get all events - ArrayList listOfUDRs= new ArrayList<>(); - QueryBuilder parameterQuery = new QueryBuilder(dbName).where("clientId", clientId). + ArrayList listOfUDRs= new ArrayList<>(); + QueryBuilder parameterQuery = new QueryBuilder(dbName). and("instanceId", instanceId).timeTo(toMills, MILLISECONDS).timeFrom(fromMills, MILLISECONDS); generatedEvents.addAll(Time.normaliseInfluxDB(influxDBClient.executeQuery(parameterQuery))); // generate last event - generatedEvents.add(getEventBeforeTime(toMills, clientId, instanceId)); + generatedEvents.add(getEventBeforeTime(toMills, instanceId)); Map lastEventInScope = new HashMap<>(); for (Map event : generatedEvents) { if ((!lastEventInScope.isEmpty())) { Long eventTime = Double.valueOf(event.get("time").toString()).longValue(); Long eventLastTime = Double.valueOf(lastEventInScope.get("time").toString()).longValue(); - listOfUDRs.add(new OpenStackEventUDR(eventLastTime, clientId, - instanceId, lastEventInScope.get("status").toString(), - (double) (eventTime - eventLastTime) /1000)); //Seconds instead of milliseconds + listOfUDRs.add(generateValue(eventTime, eventLastTime, lastEventInScope, instanceId)); } lastEventInScope = event; } @@ -225,8 +210,8 @@ private ArrayList generateUDR(String clientId, String instanc return null; } - private Map getEventBeforeTime(Long time, String clientId, String instanceId){ - QueryBuilder parameterQuery = new QueryBuilder(dbName).where("clientId", clientId). + private Map getEventBeforeTime(Long time, String instanceId){ + QueryBuilder parameterQuery = new QueryBuilder(dbName). and("instanceId", instanceId).timeTo(time, MILLISECONDS); try { influxDBClient.executeQuery(parameterQuery); @@ -240,14 +225,7 @@ private Map getEventBeforeTime(Long time, String clientId, String instanceId){ } private DateTime whenWasLastPull() { - DateTime last; - - LatestPullORM pull = (LatestPullORM) HibernateClient.getInstance().getObject(LatestPullORM.class, 1l); - if (pull == null) { - last = new DateTime(0); - } else { - last = new DateTime(pull.getTimeStamp()); - } + DateTime last = getLatestPull(); SchedulerLogger.log("Getting the last pull date " + last.toString()); // get date specified by admin String date = settings.getOpenstackFirstImport(); diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/openstack/NeutronUDRRunner.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/openstack/NeutronUDRRunner.java new file mode 100644 index 0000000..bac09be --- /dev/null +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/openstack/NeutronUDRRunner.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.schedule.runner.openstack; + +import ch.icclab.cyclops.consume.data.mapping.udr.OpenStackIpTimeUDR; +import ch.icclab.cyclops.consume.data.mapping.udr.OpenStackUDR; +import ch.icclab.cyclops.load.Loader; +import ch.icclab.cyclops.persistence.HibernateClient; +import ch.icclab.cyclops.persistence.pulls.LatestPullNeutron; +import ch.icclab.cyclops.schedule.runner.OpenStackClient; +import ch.icclab.cyclops.util.loggers.SchedulerLogger; +import org.joda.time.DateTime; + +import java.util.Map; + +/** + * Author: Oleksii Serhiienko + * Updated on: 26-Aug-16 + * Description: Runner to generate Neutron udr records out of events and send to the queue + */ +public class NeutronUDRRunner extends OpenStackClient { + public String getDbName() { + return Loader.getSettings().getOpenstackSettings().getOpenstackEventNeutronTable(); + } + + public OpenStackUDR generateValue(Long eventTime, Long eventLastTime, Map lastEventInScope, String instanceId) { + return new OpenStackIpTimeUDR(eventLastTime / 1000, lastEventInScope.get("account").toString(), + instanceId, lastEventInScope.get("type").toString(), + (double) (eventTime - eventLastTime) / 1000); //Seconds instead of milliseconds; + } + + public void updateLatestPull(Long time){ + LatestPullNeutron pull = (LatestPullNeutron) hibernateClient.getObject(LatestPullNeutron.class, 1l); + if (pull == null) { + pull = new LatestPullNeutron(time); + } else { + pull.setTimeStamp(time); + } + SchedulerLogger.log("The last pull set to "+pull.getTimeStamp().toString()); + hibernateClient.persistObject(pull); + } + + public DateTime getLatestPull(){ + DateTime last; + LatestPullNeutron pull = (LatestPullNeutron) HibernateClient.getInstance().getObject(LatestPullNeutron.class, 1l); + if (pull == null) { + last = new DateTime(0); + } else { + last = new DateTime(pull.getTimeStamp()); + } + return last; + } +} \ No newline at end of file diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/openstack/NovaUDRRunner.java b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/openstack/NovaUDRRunner.java new file mode 100644 index 0000000..3558a09 --- /dev/null +++ b/collectors/openstack-events/src/main/java/ch/icclab/cyclops/schedule/runner/openstack/NovaUDRRunner.java @@ -0,0 +1,70 @@ + +/* + * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package ch.icclab.cyclops.schedule.runner.openstack; + +import ch.icclab.cyclops.consume.data.mapping.udr.OpenStackUDR; +import ch.icclab.cyclops.consume.data.mapping.udr.OpenStackUpTimeUDR; +import ch.icclab.cyclops.load.Loader; +import ch.icclab.cyclops.persistence.HibernateClient; +import ch.icclab.cyclops.persistence.pulls.LatestPullNova; +import ch.icclab.cyclops.schedule.runner.OpenStackClient; +import ch.icclab.cyclops.util.loggers.SchedulerLogger; +import org.joda.time.DateTime; + +import java.util.Map; + +/** + * Author: Oleksii Serhiienko + * Updated on: 26-Aug-16 + * Description: Runner to generate Nova udr records out of events and send to the queue + */ +public class NovaUDRRunner extends OpenStackClient { + public String getDbName() { + return Loader.getSettings().getOpenstackSettings().getOpenstackEventNovaTable(); + } + + public OpenStackUDR generateValue(Long eventTime, Long eventLastTime, Map lastEventInScope, String instanceId){ + return new OpenStackUpTimeUDR(eventLastTime /1000, lastEventInScope.get("account").toString(), + instanceId, lastEventInScope.get("type").toString(), + (double) (eventTime - eventLastTime) /1000, //Seconds instead of milliseconds + new Double (lastEventInScope.get("cpu").toString()), + new Double (lastEventInScope.get("memory").toString())); + } + + public void updateLatestPull(Long time){ + LatestPullNova pull = (LatestPullNova) hibernateClient.getObject(LatestPullNova.class, 1l); + if (pull == null) { + pull = new LatestPullNova(time); + } else { + pull.setTimeStamp(time); + } + SchedulerLogger.log("The last pull set to "+pull.getTimeStamp().toString()); + hibernateClient.persistObject(pull); + } + + public DateTime getLatestPull(){ + DateTime last; + LatestPullNova pull = (LatestPullNova) HibernateClient.getInstance().getObject(LatestPullNova.class, 1l); + if (pull == null) { + last = new DateTime(0); + } else { + last = new DateTime(pull.getTimeStamp()); + } + return last; + } +} diff --git a/core/billing/ReleaseNotes.txt b/core/billing/ReleaseNotes.txt new file mode 100644 index 0000000..20d30c4 --- /dev/null +++ b/core/billing/ReleaseNotes.txt @@ -0,0 +1,7 @@ +Version: 2.1.0 +Date: 16/September/2016 +Notes: Data ingestion and query execution optimisations, added support for new CDR envelope (for LocalBillRequest command) + +Version: 2.0.0 +Date: 1/July/2016 +Notes: Initial release of the RCB Cyclops Billing micro service \ No newline at end of file diff --git a/core/billing/bin/billing.jar b/core/billing/bin/billing.jar index 0a377c6..6c8ffe8 100644 Binary files a/core/billing/bin/billing.jar and b/core/billing/bin/billing.jar differ diff --git a/core/billing/config/billing.conf b/core/billing/config/billing.conf index 3392b1f..e1d27eb 100644 --- a/core/billing/config/billing.conf +++ b/core/billing/config/billing.conf @@ -6,7 +6,7 @@ InfluxDBHost=localhost InfluxDBUsername=root InfluxDBPassword=root InfluxDBPort=8086 -InfluxDBTSDB=cyclops.cdr +InfluxDBTSDB=cyclops.billing InfluxDBDefaultMeasurement=unknown InfluxDBPageSizeLimit=500 diff --git a/core/billing/pom.xml b/core/billing/pom.xml index ce1b393..57e73de 100644 --- a/core/billing/pom.xml +++ b/core/billing/pom.xml @@ -7,19 +7,20 @@ ch.icclab.cyclops.billing cyclops-billing jar - 0.0.1 + 2.1.0 Billing 1.8 UTF-8 UTF-8 - 2.3.4 - 2.9.3 - 2.5 - 3.6.1 - 1.7.12 + 2.3.7 + 2.9.4 + 3.1.0 + 3.6.5 + 1.7.21 2.2 + 2.6.2 @@ -44,6 +45,7 @@ org.apache.maven.plugins maven-assembly-plugin + 2.6 jar-with-dependencies @@ -102,7 +104,7 @@ javax.servlet - servlet-api + javax.servlet-api ${servlet-version} @@ -118,47 +120,47 @@ org.slf4j jul-to-slf4j - 1.7.12 + ${slf4j-version} runtime org.apache.logging.log4j log4j-api - 2.3 + ${log4j-version} org.apache.logging.log4j log4j-core - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-web - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-jul - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-slf4j-impl - 2.3 + ${log4j-version} runtime org.json json - 20140107 + 20160212 com.google.code.gson gson - 2.6.2 + 2.7 io.gsonfire @@ -178,17 +180,32 @@ com.metapossum metapossum-scanner - 1.0 + 1.0.1 com.github.wnameless json-flattener - 0.1.6 + 0.2.2 commons-beanutils commons-beanutils 1.9.2 + + org.apache.httpcomponents + httpclient + 4.5.2 + + + com.lmax + disruptor + 3.3.5 + + + org.atteo + evo-inflector + 1.2.1 + \ No newline at end of file diff --git a/core/billing/scripts/compile.sh b/core/billing/scripts/compile.sh index cab81ac..519cf68 100755 --- a/core/billing/scripts/compile.sh +++ b/core/billing/scripts/compile.sh @@ -26,4 +26,4 @@ mvn dependency:tree mvn package assembly:single cd target -mv cyclops-billing-0.0.1-jar-with-dependencies.jar ../bin/billing.jar +mv cyclops-billing-2.1.0-jar-with-dependencies.jar ../bin/billing.jar diff --git a/core/billing/src/main/java/ch/icclab/cyclops/application/Main.java b/core/billing/src/main/java/ch/icclab/cyclops/application/Main.java index 514e0af..bd8cf0c 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/application/Main.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/application/Main.java @@ -21,6 +21,7 @@ import ch.icclab.cyclops.consume.data.DataConsumer; import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.Settings; +import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.load.model.PublisherCredentials; import ch.icclab.cyclops.publish.RabbitMQPublisher; import ch.icclab.cyclops.timeseries.InfluxDBClient; @@ -34,9 +35,14 @@ /** * Author: Skoviera * Created: 26/04/16 - * Description: Entry point for BOX micro service + * Description: Entry point for Billing micro service */ -public class Main extends Application{ +public class Main extends Application { + + static { + // Nothing can appear before this initializer + System.setProperty("Log4jContextSelector", "org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"); + } final static Logger logger = LogManager.getLogger(Main.class.getName()); @@ -55,10 +61,10 @@ public class Main extends Application{ "RCB Cyclops: Billing micro service", "Author: Martin Skoviera, ICCLab ZHAW", "", "Required parameters: path to configuration file", - "Optional parameters: HTTP port","", + "Optional parameters: HTTP port", "", "Example: java -jar billing.jar config.txt 4569"); - public static void main(String[] args){ + public static void main(String[] args) { outputProgressBar("Loading RCB Cyclops Billing micro service "); // check params and potentially stop execution @@ -92,6 +98,7 @@ public static void main(String[] args){ /** * Check number of parameters + * * @param args as string array */ private static void checkParameters(String[] args) { @@ -107,6 +114,7 @@ private static void checkParameters(String[] args) { /** * Check whether parameter was help + * * @param param to be examined */ private static void checkHelp(String param) { @@ -120,6 +128,7 @@ private static void checkHelp(String param) { /** * Make sure configuration file is valid + * * @param param path */ private static void checkConfigurationFile(String param) { @@ -199,6 +208,7 @@ private static void checkAndSetupRabbitMQ() { /** * Make sure ports are valid + * * @param component to be bind */ private static void bindServer(Component component) { @@ -230,6 +240,7 @@ private static void bindServer(Component component) { /** * Check port specification + * * @param args to be processed */ private static void checkCustomPortOption(String[] args) { @@ -239,7 +250,8 @@ private static void checkCustomPortOption(String[] args) { Loader.getSettings().getServerSettings().setServerHTTPPort(port); logger.trace(String.format("Custom port specified as %d, configuration file will be ignored", port)); - } catch (Exception ignored){} + } catch (Exception ignored) { + } } outputProgressBar(); @@ -250,8 +262,13 @@ private static void checkCustomPortOption(String[] args) { */ private static void checkAndConfigureInfluxDB() { try { - logger.trace("Binding to InfluxDB"); - InfluxDBClient.createInstance(Loader.getSettings().getInfluxDBCredentials()); + logger.trace("Binding to InfluxDB and creating databases"); + InfluxDBCredentials credentials = Loader.getSettings().getInfluxDBCredentials(); + InfluxDBClient client = new InfluxDBClient(credentials); + + client.ping(); + client.createDatabases(credentials.getInfluxDBTSDB()); + } catch (Exception e) { String log = String.format("Couldn't connect to InfluxDb: %s", e.getMessage()); logger.error(log); @@ -282,11 +299,11 @@ private static void outputProgressBar() { outputProgressBar("..."); } - private static void outputProgressBar(Boolean ... emptyLine) { + private static void outputProgressBar(Boolean... emptyLine) { outputProgressBar("...", emptyLine); } - private static void outputProgressBar(String text, Boolean ... emptyLine) { + private static void outputProgressBar(String text, Boolean... emptyLine) { if (emptyLine.length > 0 && emptyLine[0]) { System.out.println(); } @@ -297,4 +314,4 @@ private static void outputProgressBar(String text, Boolean ... emptyLine) { System.out.println(); } } -} +} \ No newline at end of file diff --git a/core/billing/src/main/java/ch/icclab/cyclops/application/Service.java b/core/billing/src/main/java/ch/icclab/cyclops/application/Service.java index 34cc556..e906f1c 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/application/Service.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/application/Service.java @@ -18,12 +18,16 @@ package ch.icclab.cyclops.application; import ch.icclab.cyclops.endpoint.*; -import ch.icclab.cyclops.util.APICallCounter; +import ch.icclab.cyclops.util.RegexParser; +import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.restlet.Restlet; import org.restlet.routing.Router; +import java.util.ArrayList; +import java.util.List; + /** * Author: Skoviera * Created: 21/01/16 @@ -36,63 +40,46 @@ public class Service { // Router for registering api endpoints private Router router; - // API counter - private APICallCounter counter; - /** - * Attaching endpoints to router + * This method handles the incoming request and routes it to the appropriate resource class */ - private void attachTheRest() { - logger.trace("Attaching measurement endpoint"); - router.attach(String.format("%s/{%s}", MeasurementEndpoint.ENDPOINT, MeasurementEndpoint.ATTRIBUTE), MeasurementEndpoint.class); - counter.registerEndpoint(MeasurementEndpoint.ENDPOINT); - - router.attach(MeasurementsEndpoint.ENDPOINT, MeasurementsEndpoint.class); - counter.registerEndpoint(MeasurementsEndpoint.ENDPOINT); + public Restlet createInboundRoot() throws Exception { - logger.trace("Attaching data endpoint"); - router.attach(DataEndpoint.ENDPOINT, DataEndpoint.class); - counter.registerEndpoint(DataEndpoint.ENDPOINT); + logger.trace("Initialising Billing microservice and creating routes"); - logger.trace("Attaching command endpoint"); - router.attach(CommandEndpoint.ENDPOINT, CommandEndpoint.class); - counter.registerEndpoint(CommandEndpoint.ENDPOINT); - } + router = attachRoutes(); - /** - * Construct application by accessing context, creating router and counter - */ - private void initialiseApplication() { - logger.trace("Initialising Billing microservice"); + logger.trace("Routes for Billing microservice successfully created"); - router = new Router(); - counter = APICallCounter.getInstance(); + return router; } /** - * This method handles the incoming request and routes it to the appropriate resource class + * Attach routes that extend AbstractEndpoint class + * @return router */ - public Restlet createInboundRoot() throws Exception { - - // let's start by initialising and loading configuration settings - initialiseApplication(); - - logger.trace("Creating routes for Billing microservice"); - - // root, status and list endpoints - router.attach(RootEndpoint.ENDPOINT, RootEndpoint.class); - counter.registerEndpoint(RootEndpoint.ENDPOINT); - - router.attach(StatusEndpoint.ENDPOINT, StatusEndpoint.class); - counter.registerEndpointWithoutCounting(StatusEndpoint.ENDPOINT); - - router.attach(ListEndpoint.ENDPOINT, ListEndpoint.class); - counter.registerEndpointWithoutCounting(ListEndpoint.ENDPOINT); - - // attach other endpoints - attachTheRest(); - - logger.trace("Routes for Billing microservice successfully created"); + private Router attachRoutes() { + Router router = new Router(); + + List list = new ArrayList<>(); + + try { + // find all endpoints and add them to the list + list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) -> !AbstractEndpoint.class.getSimpleName(). + equals(RegexParser.getFileName(fileName))).scan(AbstractEndpoint.class.getPackage().getName())); + } catch (Exception ignored) {} + + // create routes + for (Class clazz : list) { + // only those which extend Endpoint + if (AbstractEndpoint.class.isAssignableFrom(clazz)) { + try { + // get endpoint's route path + String route = ((AbstractEndpoint) clazz.newInstance()).getRoute(); + router.attach(route, clazz); + } catch (Exception ignored) {} + } + } return router; } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/Command.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/Command.java index 5419bb0..6616256 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/Command.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/Command.java @@ -27,8 +27,9 @@ public abstract class Command { /** * Every command has to implement execute method + * @return Object of your selection or Null */ - protected abstract void execute(); + protected abstract Object execute(); //===== Getters and Setters public String get_class() { diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java index f3f5b83..3c01713 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java @@ -10,6 +10,8 @@ */ public class CommandConsumer extends AbstractConsumer { + private Object response; + private ExecutionStatus status; public class ExecutionStatus { private Boolean executed; @@ -42,7 +44,7 @@ public void consume(String content) { if (command != null) { try { - command.execute(); + response = command.execute(); status = new ExecutionStatus(true, String.format("[OK] command \"%s\" successfully executed", command.get_class())); CommandLogger.log(status.getMessage()); @@ -59,4 +61,7 @@ public void consume(String content) { public ExecutionStatus getStatus() { return status; } + public Object getResponse() { + return response; + } } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java index 66ed0e2..2070dd0 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java @@ -24,6 +24,7 @@ import io.gsonfire.TypeSelector; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.List; @@ -37,25 +38,25 @@ public class CommandMapping { final static Logger logger = LogManager.getLogger(CommandMapping.class.getName()); private static final Gson gson = new GsonFireBuilder() - .registerTypeSelector(Command.class, new TypeSelector() { - @Override - public Class getClassForElement(JsonElement jsonElement) { - try { + .registerTypeSelector(Command.class, new TypeSelector() { + @Override + public Class getClassForElement(JsonElement jsonElement) { + try { - String clazz = jsonElement.getAsJsonObject().get(Command.FIELD_FOR_MAPPING).getAsString(); + String clazz = jsonElement.getAsJsonObject().get(Command.FIELD_FOR_MAPPING).getAsString(); - // recursively find correct classes - List list = new ArrayList<>(); - list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) - -> clazz.equals(RegexParser.getFileName(fileName))).scan(CommandMapping.class.getPackage().getName())); + // recursively find correct classes + List list = new ArrayList<>(); + list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) + -> clazz.equals(RegexParser.getFileName(fileName))).scan(CommandMapping.class.getPackage().getName())); - // and use the first one - return (Class) Class.forName(list.get(0).getName()); - } catch (Exception e) { - return null; + // and use the first one + return (Class) Class.forName(list.get(0).getName()); + } catch (Exception e) { + return null; + } } - } - }).createGson(); + }).createGson(); /** * Map object to provided class diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/GenericBillRequest.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/GenericBillRequest.java deleted file mode 100644 index 5d73929..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/GenericBillRequest.java +++ /dev/null @@ -1,44 +0,0 @@ -package ch.icclab.cyclops.consume.command.model; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.consume.command.Command; -import ch.icclab.cyclops.publish.Messenger; - -/** - * Author: Skoviera - * Created: 29/04/16 - * Description: Command representing External bill request - */ -public class GenericBillRequest extends Command { - - private String subject; - - @Override - protected void execute() { - // send this request to Rule Engine via queue - Messenger.getInstance().broadcast(this); - } - - //====== Getters and Setters - public String getSubject() { - return subject; - } - public void setSubject(String subject) { - this.subject = subject; - } -} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/BillRequest.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/BillRequest.java new file mode 100644 index 0000000..cd04a85 --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/BillRequest.java @@ -0,0 +1,83 @@ +package ch.icclab.cyclops.consume.command.model.generic; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.consume.command.Command; +import ch.icclab.cyclops.consume.command.model.generic.model.FlushData; +import ch.icclab.cyclops.publish.APICaller; +import ch.icclab.cyclops.util.loggers.CommandLogger; + +import java.net.URL; +import java.util.List; + +/** + * Author: Skoviera + * Created: 09/09/16 + * Description: Generic Bill Request + */ +public class BillRequest extends Command { + + // TODO add service discovery + private static String CDR_URL = "localhost:4568"; + private static String COIN_URL = "localhost:4571"; + + // account holder for the bill + private String account; + + // linked accounts in federated billing + private List linked; + + // time boundaries for the bill + private Long from; + private Long to; + + @Override + protected Object execute() { + try { + // sanity checks first + if (from == null || to == null || from < 0l || to <= from) + return "[ERROR] invalid FROM and TO"; + if (account == null || account.isEmpty()) + return "[ERROR] invalid account name"; + + CommandLogger.log("Received Bill Request that is meant for Billing Rule engine, therefore synchronously forwarding it"); + + // perform CDR flush + new APICaller().post(new URL(String.format("http://%s/command", CDR_URL)), new FlushData(from, to, account, linked)); + + // request bill from Coin + APICaller.Response response = new APICaller().post(new URL(String.format("http://%s/ruleengine/facts", COIN_URL)), this); + return response.getAsList(); + + } catch (Exception e) { + return "No data"; + } + } + + public String getAccount() { + return account; + } + public List getLinked() { + return linked; + } + public Long getFrom() { + return from; + } + public Long getTo() { + return to; + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/LocalBillRequest.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/LocalBillRequest.java new file mode 100644 index 0000000..09bb946 --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/LocalBillRequest.java @@ -0,0 +1,83 @@ +package ch.icclab.cyclops.consume.command.model.generic; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.consume.command.Command; +import ch.icclab.cyclops.consume.command.model.generic.model.Bill; +import ch.icclab.cyclops.consume.command.model.generic.model.CDR; +import ch.icclab.cyclops.consume.command.model.generic.model.FlushData; +import ch.icclab.cyclops.publish.APICaller; +import ch.icclab.cyclops.util.loggers.CommandLogger; + +import java.net.URL; +import java.util.List; + +/** + * Author: Skoviera + * Created: 14/09/16 + * Description: Local bill request that doesn't call Rule engine + */ +public class LocalBillRequest extends Command { + + // TODO add service discovery + private static String CDR_URL = "localhost:4568"; + + // account holder for the bill + private String account; + + // linked accounts in federated billing + private List linked; + + // time boundaries for the bill + private Long from; + private Long to; + + @Override + protected Object execute() { + try { + // sanity checks first + if (from == null || to == null || from < 0l || to <= from) + return "[ERROR] invalid FROM and TO"; + if (account == null || account.isEmpty()) + return "[ERROR] invalid account name"; + + CommandLogger.log("Received Local Bill Request, thus asking CDR for data and generating a bill locally"); + + // prepare flush command + FlushData flush = new FlushData(from, to, account, linked); + flush.disableSync(); + flush.enableOutput(); + + // send flush data request + APICaller.Response response = new APICaller().post(new URL(String.format("http://%s/command", CDR_URL)), flush); + + // parse as list of CDRs + List cdrs = response.getAsListOfType(CDR.class); + + // create a bill + Bill bill = new Bill(account, from, to); + + // process received CDR records + bill.addAndProcessCDRs(cdrs); + + return bill; + + } catch (Exception e) { + return "No data"; + } + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/Bill.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/Bill.java new file mode 100644 index 0000000..9a20c7c --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/Bill.java @@ -0,0 +1,76 @@ +package ch.icclab.cyclops.consume.command.model.generic.model; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.util.BeanList; + +import java.util.ArrayList; +import java.util.List; + +/** + * Author: Skoviera + * Created: 09/09/16 + * Description: Generic bill container + */ +public class Bill { + private String _class = getClass().getSimpleName(); + private String account; + private Long from; + private Long to; + private Double charge; + private List records; + + public Bill(String account, Long from, Long to) { + this.account = account; + this.from = from; + this.to = to; + this.charge = 0d; + this.records = new ArrayList<>(); + } + + /** + * Process individual Charge Records from list of CDRs + * @param cdrs + */ + public void addAndProcessCDRs(List cdrs) { + if (cdrs != null && !cdrs.isEmpty()) { + List data = new ArrayList<>(); + + // flatten the structure and get all ChargeData records + cdrs.stream().filter(cdr -> cdr.getData() != null).forEach(item -> data.addAll(BeanList.populate(item.getData(), ChargeData.class))); + + // iterate over charge data records + for (ChargeData record: data) { + + int index = records.indexOf(record); + + if (index >= 0) { + // update found record + ChargeData tmp = records.get(index); + tmp.setCharge(tmp.getCharge() + record.getCharge()); + tmp.setUsage(tmp.getUsage() + record.getUsage()); + } else { + // add new record + records.add(record); + } + + // update overall counter + charge += record.getCharge(); + } + } + } +} diff --git a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OpenStackEventUDR.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/CDR.java similarity index 56% rename from collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OpenStackEventUDR.java rename to core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/CDR.java index e217b9c..4fe2a99 100644 --- a/collectors/openstack-events/src/main/java/ch/icclab/cyclops/consume/data/mapping/OpenStackEventUDR.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/CDR.java @@ -1,3 +1,4 @@ +package ch.icclab.cyclops.consume.command.model.generic.model; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -14,73 +15,70 @@ * License for the specific language governing permissions and limitations * under the License. */ -package ch.icclab.cyclops.consume.data.mapping; + +import java.util.List; +import java.util.Map; /** - * Author: Oleksii - * Date: 01/06/2016 - * Description: This class holds the OpenStackEventUDR response + * Author: Skoviera + * Created: 06/09/16 + * Description: CDR output */ - -public class OpenStackEventUDR { - - public OpenStackEventUDR(Long time, String account, String instanceId, String status, Double usage){ - this.time = time; - this.account = account; - this.instanceId = instanceId; - this.status = status; - this.usage = usage; - } - +public class CDR { private String _class = getClass().getSimpleName(); - private Long time; private String account; - private String instanceId; - private String status; - private Double usage; + private Long time; + private Long from; + private Long to; + private Double charge; + private List data; public String get_class() { return _class; } - - public Long getTime() { - return time; - } - - public void setTime(Long time) { - this.time = time; + public void set_class(String _class) { + this._class = _class; } public String getAccount() { return account; } - public void setAccount(String account) { this.account = account; } - public String getInstanceId() { - return instanceId; + public Long getTime() { + return time; } - - public void setInstanceId(String instanceId) { - this.instanceId = instanceId; + public void setTime(Long time) { + this.time = time; } - - public String getStatus() { - return status; + public Long getFrom() { + return from; + } + public void setFrom(Long from) { + this.from = from; } - public void setStatus(String status) { - this.status = status; + public Long getTo() { + return to; + } + public void setTo(Long to) { + this.to = to; } - public Double getUsage() { - return usage; + public Double getCharge() { + return charge; + } + public void setCharge(Double charge) { + this.charge = charge; } - public void setUsage(Double usage) { - this.usage = usage; + public List getData() { + return data; + } + public void setData(List data) { + this.data = data; } } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/ChargeData.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/ChargeData.java new file mode 100644 index 0000000..da1653d --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/ChargeData.java @@ -0,0 +1,86 @@ +package ch.icclab.cyclops.consume.command.model.generic.model; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.base.Objects; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 08/09/16 + * Description: Charge data for CDR record + */ +public class ChargeData { + private String _class; + private String account; + private Double usage; + private Double charge; + private Map metadata; + + public ChargeData() { + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ChargeData that = (ChargeData) o; + return Objects.equal(_class, that._class) && + Objects.equal(account, that.account) && + Objects.equal(metadata, that.metadata); + } + + @Override + public int hashCode() { + return Objects.hashCode(_class, account, metadata); + } + + public String get_class() { + return _class; + } + public void set_class(String _class) { + this._class = _class; + } + + public String getAccount() { + return account; + } + public void setAccount(String account) { + this.account = account; + } + + public Double getUsage() { + return usage; + } + public void setUsage(Double usage) { + this.usage = usage; + } + + public Double getCharge() { + return charge; + } + public void setCharge(Double charge) { + this.charge = charge; + } + + public Map getMetadata() { + return metadata; + } + public void setMetadata(Map metadata) { + this.metadata = metadata; + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/FlushData.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/FlushData.java new file mode 100644 index 0000000..f31fc8d --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/FlushData.java @@ -0,0 +1,60 @@ +package ch.icclab.cyclops.consume.command.model.generic.model; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import java.util.ArrayList; +import java.util.List; + +/** + * Author: Skoviera + * Created: 14/09/16 + * Description: Flush Data request + */ +public class FlushData { + private String _class = getClass().getSimpleName(); + private List accounts; + private Long from; + private Long to; + private Boolean sync; + private Boolean output; + + public FlushData(Long from, Long to, String account, List accounts) { + this.from = from; + this.to = to; + + this.accounts = new ArrayList<>(); + this.accounts.add(account); + + // in case we have federated bill request + if (accounts != null && !accounts.isEmpty()) + this.accounts.addAll(accounts); + } + + public void enableSync() { + sync = true; + } + public void disableSync() { + sync = false; + } + + public void enableOutput() { + output = true; + } + public void disableOutput() { + output = false; + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBill.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBill.java deleted file mode 100644 index 1aa55be..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBill.java +++ /dev/null @@ -1,243 +0,0 @@ -package ch.icclab.cyclops.consume.command.model.localbill; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.load.model.InfluxDBCredentials; -import ch.icclab.cyclops.timeseries.RemoveNullValues; -import com.github.wnameless.json.flattener.JsonFlattener; -import com.google.gson.Gson; -import org.influxdb.dto.Point; -import org.joda.time.DateTime; - -import java.util.*; - -/** - * Author: Skoviera - * Created: 28/06/16 - * Description: - */ -public class LocalBill { - private String account; - private Double amount = 0d; - private Long from; - private Long to; - private List bills = new ArrayList<>(); - - private Double discount; - private Double amountBeforeDiscount; - public void applyDiscount(Double percentage) { - if (percentage != null && percentage > 0d) { - discount = percentage; - amountBeforeDiscount = amount; - amount = amount - amount * percentage / 100; - if (amount < 0d) { - amount = 0d; - } - } - } - - private Double VAT; - private Double amountBeforeVAT; - public void applyVAT(Double percentage) { - if (percentage != null && percentage > 0) { - VAT = percentage; - amountBeforeVAT = amount; - amount = amount + amount * percentage / 100; - if (amount < 0d) { - amount = 0d; - } - } - } - - public void calculateAmount() { - for (BillEntry bill: bills) { - Double counter = bill.amount; - - if (bill.sla_discount != null) { - counter = counter - bill.sla_discount; - } - - if (bill.coupon_discount != null) { - counter = counter - bill.coupon_discount; - } - - Double finalised = (counter > 0)? counter: 0; - if (!Objects.equals(bill.amount, finalised)) { - bill.original_amount = bill.amount; - bill.amount = finalised; - } - - amount += bill.amount; - } - } - - public void applySLA(Map slas) { - if (slas != null && !slas.isEmpty()) { - // process all SLA violations - for (Map.Entry sla: slas.entrySet()) { - - // find appropriate bill - for (BillEntry bill: bills) { - if (bill.getBill().equals(sla.getKey())) { - bill.applySLAViolation((Double) sla.getValue()); - break; - } - } - } - } - } - - public void applyCoupons(Map coupons) { - if (coupons != null && !coupons.isEmpty()){ - // process all coupons and apply them to correct bills - for (Map.Entry coupon: coupons.entrySet()) { - - // find appropriate bill - for (BillEntry bill: bills) { - if (bill.getBill().equals(coupon.getKey())) { - bill.applyCoupon((Double) coupon.getValue()); - break; - } - } - } - } - } - - private class BillEntry { - private String bill; - private Double amount = 0d; - private Double sla_percentage; - private Double sla_discount; - private Double coupon_discount; - private Double original_amount; - - public BillEntry(String bill, LocalBillCDRMapping cdr) { - this.bill = bill; - - addCdrToStatement(cdr); - } - - public void addCdrToStatement(LocalBillCDRMapping cdr) { - amount = amount + cdr.getCharge(); - } - - public String getBill() { - return bill; - } - - public void applySLAViolation(Double percentage) { - sla_percentage = percentage; - sla_discount = amount * sla_percentage / 100; - if (sla_discount < 0d) { - sla_discount = 0d; - } else if (sla_discount > amount){ - sla_discount = amount; - } - } - - public void applyCoupon(Double value) { - coupon_discount = value; - } - } - - public LocalBill(String account, Long from, Long to) { - this.account = account; - - this.from = (from != null && from > 0)? from : 0; - this.to = (to != null && to > this.from) ? to : DateTime.now().getMillis() / 1000; - } - - /** - * Calculate and prepare a bill based on list of CDRs - * @param cdrs to be taken into account - */ - public void addToBill(List cdrs, String clazz) { - // iterate over all cdr records - for (LocalBillCDRMapping cdr: cdrs) { - // have we seen this bill before? - Boolean found = false; - for (BillEntry entry: bills) { - if (entry.getBill().equals(clazz)) { - entry.addCdrToStatement(cdr); - found = true; - } - } - - if (!found) { - // it's a new bill entry - BillEntry entry = new BillEntry(clazz, cdr); - bills.add(entry); - } - } - } - - public Point.Builder toDBPoint() { - if (bills.isEmpty()) { - bills = null; - } - - // first translate it into json - String json = new Gson().toJson(this); - - // now flatten it properly - Map flat = JsonFlattener.flattenAsMap(json); - - // let's delete account as we will use it as tag and not field - flat.remove("account"); - - // just in case remove all null values - Map valid = RemoveNullValues.fromMap(flat); - - return Point.measurement(this.getClass().getSimpleName()).tag("account", account).fields(valid); - } - - //======= Getters and Setters - public String getAccount() { - return account; - } - public void setAccount(String account) { - this.account = account; - } - - public Double getAmount() { - return amount; - } - public void setAmount(Double amount) { - this.amount = amount; - } - - public List getBills() { - return bills; - } - public void setBills(List bills) { - this.bills = bills; - } - - public Long getFrom() { - return from; - } - public void setFrom(Long from) { - this.from = from; - } - - public Long getTo() { - return to; - } - public void setTo(Long to) { - this.to = to; - } -} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBillRequest.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBillRequest.java deleted file mode 100644 index 9a2a8db..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/consume/command/model/localbill/LocalBillRequest.java +++ /dev/null @@ -1,105 +0,0 @@ -package ch.icclab.cyclops.consume.command.model.localbill; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.consume.command.Command; -import ch.icclab.cyclops.timeseries.BatchPointsContainer; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.QueryBuilder; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * Author: Skoviera - * Created: 27/06/16 - * Description: Command representing Local bill request - */ -public class LocalBillRequest extends Command { - - private static String ACCOUNT_FIELD = "account"; - - private String subject; - private List include; - private Long from; - private Long to; - - // SLAs, discounts and - private Map sla; - private Map coupons; - private Double discount; - private Double vat; - - @Override - protected void execute() { - // create local bill - LocalBill invoice = new LocalBill(subject, from, to); - - // only process this message when subject and include are valid - if (subject != null && !subject.isEmpty() && include != null && !include.isEmpty()) { - // iterate over all included items - for (String cdr: include) { - // only if we got valid cdr request - if (!cdr.isEmpty()) { - - // create query - QueryBuilder query = prepareQuery(cdr); - - // execute query - List response = InfluxDBClient.getInstance().executeQueryAndMapItToClass(query, LocalBillCDRMapping.class); - - // add it to container - if (response != null && !response.isEmpty()) { - invoice.addToBill(response, cdr); - } - } - } - } - - invoice.applySLA(sla); - invoice.applyCoupons(coupons); - invoice.calculateAmount(); - invoice.applyDiscount(discount); - invoice.applyVAT(vat); - - // proceed with storing this invoice to database (even if it's empty) - InfluxDBClient.getInstance().persistSinglePoint(invoice.toDBPoint()); - } - - /** - * Create query for InfluxDB - * @param name as subject - * @return QueryBuilder - */ - private QueryBuilder prepareQuery(String name) { - // build query for included measurement and subject - QueryBuilder query = new QueryBuilder(name).where(ACCOUNT_FIELD, subject); - - // if from is valid - if (from != null && from > 0) { - query.timeFrom(from, TimeUnit.SECONDS); - } - - // to is valid - if (to != null && to > 0) { - query.timeTo(to, TimeUnit.SECONDS); - } - - return query; - } -} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java index 4bb2e74..584efd3 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java @@ -1,24 +1,10 @@ package ch.icclab.cyclops.consume.data; import ch.icclab.cyclops.consume.AbstractConsumer; -import ch.icclab.cyclops.load.Loader; +import ch.icclab.cyclops.executor.TaskExecutor; import ch.icclab.cyclops.load.model.PublisherCredentials; -import ch.icclab.cyclops.publish.Messenger; -import ch.icclab.cyclops.timeseries.BatchPointsContainer; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.RemoveNullValues; -import ch.icclab.cyclops.util.RegexParser; -import ch.icclab.cyclops.util.loggers.DataLogger; -import com.github.wnameless.json.flattener.JsonFlattener; -import com.google.gson.Gson; -import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.influxdb.dto.Point; - -import java.math.BigDecimal; -import java.util.*; -import java.util.concurrent.TimeUnit; /** * Author: Skoviera @@ -28,371 +14,19 @@ public class DataConsumer extends AbstractConsumer { final static Logger logger = LogManager.getLogger(DataConsumer.class.getName()); - private static InfluxDBClient influxDBClient = InfluxDBClient.getInstance(); - private static Messenger messenger = Messenger.getInstance(); - - public static TimeUnit TIME_UNIT = TimeUnit.SECONDS; - private PublisherCredentials publisherSettings; private String defaultMeasurementName; + private TaskExecutor executor; - private Long validRecords; - - private BatchPointsContainer container; - private List broadcast; - private Map> dispatch; public DataConsumer(String name, PublisherCredentials settings) { this.defaultMeasurementName = name; this.publisherSettings = settings; + this.executor = TaskExecutor.getInstance(); } @Override public void consume(String content) { - initialise(); - - try { - // try to map it as array - List array = new Gson().fromJson(content, List.class); - - // iterate over object entries - array.forEach(this::processDataFrame); - - } catch (Exception ignored) { - // this means it was not an array to begin with, just a simple object - processDataFrame(content); - } - - finalise(); - } - - public Long getNumberOfValidRecords() { - return validRecords; - } - - private void initialise() { - validRecords = 0l; - container = new BatchPointsContainer(); - broadcast = new ArrayList<>(); - dispatch = new HashMap<>(); - } - - private void finalise() { - // persist points that were created - influxDBClient.persistContainer(container); - - // broadcast - if (broadcast != null && !broadcast.isEmpty()) { - if (broadcast.size() == 1) { - // broadcast it as value - messenger.broadcast(broadcast.get(0)); - } else { - // broadcast it as array - messenger.broadcast(broadcast); - } - } - - // dispatch - if (dispatch != null && !dispatch.isEmpty()) { - for (Map.Entry entry: dispatch.entrySet()) { - List list = (List) entry.getValue(); - - if (list.size() == 1) { - // publish it as value - messenger.publish(list.get(0), (String) entry.getKey()); - } else { - // publish it as array - messenger.publish(list, (String) entry.getKey()); - } - } - } - } - - /** - * Process data frame as OBJECT - * @param obj to be processed - */ - private void processDataFrame(Map obj) { - storeAndPublish(obj); - } - - /** - * Process data frame as STRING - * @param str to be processed - */ - private void processDataFrame(String str) { - try { - Map obj = new Gson().fromJson(str, Map.class); - - storeAndPublish(obj); - } catch (Exception ignored) { - // if incoming JSON is not valid we are simply skipping it - } - } - - /** - * Process, store and publish incoming object - * @param obj representation - */ - private void storeAndPublish(Map obj) { - try { - // now parse it and create a database Point - ConsumedData data = processDataAndGetPoint(obj); - - // and finally persist it - container.addPoint(data.getPoint()); - - // publish or broadcast if desired - if (data.shouldPublish()) { - - // reapply class definition - obj.put(DataMapping.FIELD_FOR_MAPPING, data.getClazz()); - - // where to publish - if (data.doNotBroadcastButRoute()) { - String clazz = data.getClazz(); - - // add to internal dispatch structure - if (dispatch.containsKey(clazz)) { - // we need to fetch the list - List list = dispatch.get(clazz); - list.add(obj); - } else { - // we need to create a new one - List list = new ArrayList<>(); - list.add(obj); - - dispatch.put(clazz, list); - } - - } else { - broadcast.add(obj); - } - } - - validRecords += 1; - - } catch (Exception notValidJson) { - DataLogger.log(String.format("Received event/measurement is not a valid JSON: %s", notValidJson.getMessage())); - } - } - - /** - * Data holder for consumed data including database point and mapping guidelines - */ - private class ConsumedData { - private String clazz; - private Point.Builder builder; - private Boolean shouldPublish; - private Boolean doNotBroadcastButRoute; - - public ConsumedData(String clazz, Point.Builder builder, Boolean shouldPublish, Boolean doNotBroadcastButRoute) { - this.clazz = clazz; - this.builder = builder; - this.shouldPublish = shouldPublish; - this.doNotBroadcastButRoute = doNotBroadcastButRoute; - } - - public ConsumedData(String clazz, Point.Builder builder, DataMapping guideline) { - this.clazz = clazz; - this.builder = builder; - this.shouldPublish = guideline.shouldPublish(); - this.doNotBroadcastButRoute = guideline.doNotBroadcastButRoute(); - } - - public String getClazz() { - return clazz; - } - public Point.Builder getPoint() { - return builder; - } - public Boolean shouldPublish() { - return shouldPublish; - } - public Boolean doNotBroadcastButRoute() { - return doNotBroadcastButRoute; - } - } - - /** - * Process and create database Point - * @param map to be processed - * @return point - */ - private ConsumedData processDataAndGetPoint(Map map) { - try { - // check whether type field is present - String clazz = (String) map.get(DataMapping.FIELD_FOR_MAPPING); - - // find corresponding classes - Set set = new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) - -> clazz.equals(RegexParser.getFileName(fileName))).scan(DataConsumer.class.getPackage().getName()); - - // move to the first element - Optional first = set.stream().findFirst(); - - if (first.isPresent()) { - // access object based on provided class definition - DataMapping guideline = (DataMapping) Class.forName(first.get().getName()).newInstance(); - - DataLogger.log(String.format("Event/measurement received with \"%s\" guidelines", clazz)); - - // by default we will work with the original map - Map processed = map; - try { - // let's call user specified pre-processing method - Map preProcessing = guideline.preProcess(map); - - // assign new one if it's valid - if (preProcessing != null && !preProcessing.isEmpty()) { - processed = preProcessing; - } - } catch (Exception ignored) { - } - - // we will be storing flattened version of the map - Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(processed)); - - // remove type from original HashMap - flat.remove(DataMapping.FIELD_FOR_MAPPING); - - // extract tags from flattened json - Map tags = extractTags(flat, guideline.getTagNames()); - - // parse time field if it was provided - String timeField = guideline.getTimeField(); - Long timeStamp = (timeField != null && flat.containsKey(timeField))? getTimeStamp(flat.get(timeField)): null; - if (timeStamp != null && timeStamp >= 0) flat.remove(timeField); - - // determine correct time unit - TimeUnit unit = guideline.getTimeUnit(); - if (unit == null) { - unit = TIME_UNIT; - } - // finally construct database point - Point.Builder builder = constructPoint(clazz, timeStamp, unit, flat, tags); - return new ConsumedData(clazz, builder, guideline); - } else { - // guideline was not present or wasn't valid - throw new Exception(); - } - - } catch (Exception withoutGuidelines){ - // mapping failed, guideline was not provided, use default - DataLogger.log("Event/measurement received without valid guidelines"); - - Point.Builder builder; - - Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(map)); - - // even though we don't have mapping template, we still want to have proper measurement name if TYPE is present - if (flat.containsKey(DataMapping.FIELD_FOR_MAPPING)){ - - // get that name - String measurement = (String) flat.get(DataMapping.FIELD_FOR_MAPPING); - - // remove it from flat - flat.remove(DataMapping.FIELD_FOR_MAPPING); - - // construct point - builder = constructPoint(measurement, flat); - } else { - builder = constructPoint(defaultMeasurementName, flat); - } - - // determine publisher preferences - Boolean shouldPublish = publisherSettings.getPublisherIncludeAlsoUnknown(); - Boolean doNotRouteButBroadcast = publisherSettings.getPublisherByDefaultDispatchInsteadOfBroadcast(); - String clazz = Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement(); - - return new ConsumedData(clazz, builder, shouldPublish, doNotRouteButBroadcast); - } - } - - /** - * Construct database Point - * @param measurementName to be used - * @param fields as map - * @return point builder - */ - private Point.Builder constructPoint(String measurementName, Map fields) { - return Point.measurement(measurementName).fields(RemoveNullValues.fromMap(fields)); - - } - - /** - * Construct database Point - * @param measurementName to be used - * @param timeStamp to be marked - * @param unit of time to be used - * @param fields as map - * @param tags as map - * @return point builder - */ - private Point.Builder constructPoint(String measurementName, Long timeStamp, TimeUnit unit, Map fields, Map tags) { - Point.Builder builder = Point.measurement(measurementName); - - // apply time stamp with time unit - if (timeStamp != null && timeStamp >= 0 && unit != null) { - builder.time(timeStamp, unit); - } - - // apply specified tags - if (tags != null && !tags.isEmpty()) { - builder.tag(RemoveNullValues.fromMap(tags)); - } - - // and finally add fields - builder.fields(RemoveNullValues.fromMap(fields)); - - return builder; - } - - /** - * Move tags from content into variable based on provided list of tags - * @param content to be extracted - * @param tagList definition - * @return extracted tags or null - */ - private Map extractTags(Map content, List tagList) { - if (tagList != null) { - Map tags = new HashMap<>(); - - // iterate over list of tags and move them from the original content - for (String tag: tagList) { - tags.put(tag, (String) content.get(tag)); - content.remove(tag); - } - - return tags; - } else { - return null; - } - } - - /** - * Parse long from unknown type - * @param unknown object - * @return Long or null - */ - private Long getTimeStamp(Object unknown) { - try { - BigDecimal dec = (BigDecimal) unknown; - return dec.longValue(); - } catch (Exception a) { - try { - return Long.parseLong((String) unknown); - } catch (Exception b) { - try { - return (Long) unknown; - } catch (Exception c) { - try { - return Long.valueOf(unknown.toString()); - } catch (Exception d) { - return null; - } - } - } - } + executor.addTask(new DataProcess(content, publisherSettings, defaultMeasurementName)); } } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java b/core/billing/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java new file mode 100644 index 0000000..25bd5c0 --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java @@ -0,0 +1,372 @@ +package ch.icclab.cyclops.consume.data; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.executor.TaskExecutor; +import ch.icclab.cyclops.load.Loader; +import ch.icclab.cyclops.load.model.PublisherCredentials; +import ch.icclab.cyclops.publish.Messenger; +import ch.icclab.cyclops.timeseries.*; +import ch.icclab.cyclops.util.RegexParser; +import ch.icclab.cyclops.util.loggers.DataLogger; +import com.github.wnameless.json.flattener.JsonFlattener; +import com.google.gson.Gson; +import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; +import org.influxdb.dto.Point; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Process data frame in runnable manner + */ +public class DataProcess implements Runnable { + + private String content; + + private static Messenger messenger = Messenger.getInstance(); + + private PublisherCredentials publisherSettings; + private String defaultMeasurementName; + + private List broadcast; + private BatchPointsContainer container; + private Map> dispatch; + + public static TimeUnit TIME_UNIT = TimeUnit.SECONDS; + + public DataProcess(String content, PublisherCredentials publisher, String measurement) { + this.publisherSettings = publisher; + this.defaultMeasurementName = measurement; + this.container = new BatchPointsContainer(); + this.broadcast = new ArrayList<>(); + this.dispatch = new HashMap<>(); + this.content = content; + } + + @Override + public void run() { + try { + // try to map it as array + List array = new Gson().fromJson(content, List.class); + + // process individual items (cannot be in parallel as we would quickly start thread-context switching) + array.stream().forEach(this::processDataFrame); + + } catch (Exception ignored) { + // this means it was not an array to begin with, just a simple object + processDataFrame(content); + } + + finalise(); + } + + private void finalise() { + + // schedule tasks + TaskExecutor executor = TaskExecutor.getInstance(); + + // persist points + if (container.size() == 1) { + // when only one point in container use shared InfluxDB session for delaying writes + executor.addTask(() -> SharedInfluxDBSession.getSession().persistSinglePoint(container.getFirstPoint())); + } else if (container.size() > 1) { + // we did batch persisting manually so we can store it immediately + executor.addTask(() -> {new InfluxDBClient().persistContainer(container);}); + } + + // broadcast + if (broadcast != null && !broadcast.isEmpty()) { + if (broadcast.size() == 1) { + // broadcast it as value + executor.addTask(() -> {messenger.broadcast(broadcast.get(0));}); + } else { + // broadcast it as array + executor.addTask(() -> {messenger.broadcast(broadcast);}); + } + } + + // dispatch + if (dispatch != null && !dispatch.isEmpty()) { + for (Map.Entry entry: dispatch.entrySet()) { + List list = (List) entry.getValue(); + + if (list.size() == 1) { + // publish it as value + executor.addTask(() -> {messenger.publish(list.get(0), (String) entry.getKey());}); + } else { + // publish it as array + executor.addTask(() -> {messenger.publish(list, (String) entry.getKey());}); + } + } + } + } + + /** + * Process data frame as OBJECT + * @param obj to be processed + */ + private void processDataFrame(Map obj) { + storeAndPublish(obj); + } + + /** + * Process data frame as STRING + * @param str to be processed + */ + private void processDataFrame(String str) { + try { + Map obj = new Gson().fromJson(str, Map.class); + + storeAndPublish(obj); + } catch (Exception ignored) { + // if incoming JSON is not valid we are simply skipping it + } + } + + /** + * Process, store and publish incoming object + * @param obj representation + */ + private void storeAndPublish(Map obj) { + try { + // now parse it and create a database Point + ConsumedData data = processDataAndGetPoint(obj); + + // and finally persist it + container.addPoint(data.getPoint()); + + // publish or broadcast if desired + if (data.shouldPublish()) { + + // where to publish + if (data.doNotBroadcastButRoute()) { + String clazz = data.getClazz(); + + // add to internal dispatch structure + if (dispatch.containsKey(clazz)) { + // we need to fetch the list + List list = dispatch.get(clazz); + list.add(obj); + } else { + // we need to create a new one + List list = new ArrayList<>(); + list.add(obj); + + dispatch.put(clazz, list); + } + + } else { + broadcast.add(obj); + } + } + + } catch (Exception notValidJson) { + DataLogger.log(String.format("Received event/measurement is not a valid JSON: %s", notValidJson.getMessage())); + } + } + + /** + * Data holder for consumed data including database point and mapping guidelines + */ + private class ConsumedData { + private String clazz; + private Point.Builder builder; + private Boolean shouldPublish; + private Boolean doNotBroadcastButRoute; + + public ConsumedData(String clazz, Point.Builder builder, Boolean shouldPublish, Boolean doNotBroadcastButRoute) { + this.clazz = clazz; + this.builder = builder; + this.shouldPublish = shouldPublish; + this.doNotBroadcastButRoute = doNotBroadcastButRoute; + } + + public ConsumedData(String clazz, Point.Builder builder, DataMapping guideline) { + this.clazz = clazz; + this.builder = builder; + this.shouldPublish = guideline.shouldPublish(); + this.doNotBroadcastButRoute = guideline.doNotBroadcastButRoute(); + } + + public String getClazz() { + return clazz; + } + public Point.Builder getPoint() { + return builder; + } + public Boolean shouldPublish() { + return shouldPublish; + } + public Boolean doNotBroadcastButRoute() { + return doNotBroadcastButRoute; + } + } + + /** + * Process and create database Point + * @param map to be processed + * @return point + */ + private ConsumedData processDataAndGetPoint(Map map) { + try { + // check whether type field is present + String clazz = (String) map.get(DataMapping.FIELD_FOR_MAPPING); + + // find corresponding classes + Set set = new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) + -> clazz.equals(RegexParser.getFileName(fileName))).scan(DataConsumer.class.getPackage().getName()); + + // move to the first element + Optional first = set.stream().findFirst(); + + if (first.isPresent()) { + // access object based on provided class definition + DataMapping guideline = (DataMapping) Class.forName(first.get().getName()).newInstance(); + + DataLogger.log(String.format("Event/measurement received with \"%s\" guidelines", clazz)); + + // by default we will work with the original map + Map processed = map; + try { + // let's call user specified pre-processing method + Map preProcessing = guideline.preProcess(map); + + // assign new one if it's valid + if (preProcessing != null && !preProcessing.isEmpty()) { + processed = preProcessing; + } + } catch (Exception ignored) { + } + + // we will be storing flattened version of the map + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(processed)); + + // extract tags from flattened json + Map tags = extractTags(flat, guideline.getTagNames()); + + // parse time field if it was provided + String timeField = guideline.getTimeField(); + Long timeStamp = (timeField != null && flat.containsKey(timeField))? TimeStamp.cast(flat.get(timeField)): null; + if (timeStamp != null && timeStamp >= 0) flat.remove(timeField); + + // determine correct time unit + TimeUnit unit = guideline.getTimeUnit(); + if (unit == null) { + unit = TIME_UNIT; + } + // finally construct database point + Point.Builder builder = constructPoint(clazz, timeStamp, unit, flat, tags); + return new ConsumedData(clazz, builder, guideline); + } else { + // guideline was not present or wasn't valid + throw new Exception(); + } + + } catch (Exception withoutGuidelines){ + // mapping failed, guideline was not provided, use default + DataLogger.log("Event/measurement received without valid guidelines"); + + Point.Builder builder; + + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(map)); + + // even though we don't have mapping template, we still want to have proper measurement name if TYPE is present + if (flat.containsKey(DataMapping.FIELD_FOR_MAPPING)){ + + // get that name + String measurement = (String) flat.get(DataMapping.FIELD_FOR_MAPPING); + + // construct point + builder = constructPoint(measurement, flat); + } else { + builder = constructPoint(defaultMeasurementName, flat); + } + + // determine publisher preferences + Boolean shouldPublish = publisherSettings.getPublisherIncludeAlsoUnknown(); + Boolean doNotRouteButBroadcast = publisherSettings.getPublisherByDefaultDispatchInsteadOfBroadcast(); + String clazz = Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement(); + + return new ConsumedData(clazz, builder, shouldPublish, doNotRouteButBroadcast); + } + } + + /** + * Construct database Point + * @param measurementName to be used + * @param fields as map + * @return point builder + */ + private Point.Builder constructPoint(String measurementName, Map fields) { + return Point.measurement(measurementName).time(System.currentTimeMillis(), TimeUnit.MILLISECONDS).fields(RemoveNullValues.fromMap(fields)); + } + + /** + * Construct database Point + * @param measurementName to be used + * @param timeStamp to be marked + * @param unit of time to be used + * @param fields as map + * @param tags as map + * @return point builder + */ + private Point.Builder constructPoint(String measurementName, Long timeStamp, TimeUnit unit, Map fields, Map tags) { + Point.Builder builder = Point.measurement(measurementName); + + // apply time stamp with time unit + if (timeStamp != null && timeStamp >= 0 && unit != null) { + builder.time(timeStamp, unit); + } else { + builder.time(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + } + + // apply specified tags + if (tags != null && !tags.isEmpty()) { + builder.tag(RemoveNullValues.fromMap(tags)); + } + + // and finally add fields + builder.fields(RemoveNullValues.fromMap(fields)); + + return builder; + } + + /** + * Move tags from content into variable based on provided list of tags + * @param content to be extracted + * @param tagList definition + * @return extracted tags or null + */ + private Map extractTags(Map content, List tagList) { + if (tagList != null) { + Map tags = new HashMap<>(); + + // iterate over list of tags and move them from the original content + for (String tag: tagList) { + tags.put(tag, (String) content.get(tag)); + content.remove(tag); + } + + return tags; + } else { + return null; + } + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkOfferingUsageData.java b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java similarity index 74% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkOfferingUsageData.java rename to core/billing/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java index 10189d1..c015654 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkOfferingUsageData.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java @@ -1,4 +1,4 @@ -package ch.icclab.cyclops.consume.data.model.cloudstack; +package ch.icclab.cyclops.endpoint; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -16,10 +16,13 @@ * under the License. */ +import org.restlet.resource.ServerResource; + /** * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack Network Offering Usage Data + * Created: 15/09/16 + * Description: Abstract Endpoint class */ -public class CloudStackNetworkOfferingUsageData extends CloudStackUsageData { +public abstract class AbstractEndpoint extends ServerResource { + public abstract String getRoute(); } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java index a37e390..bdddaee 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java @@ -18,32 +18,23 @@ package ch.icclab.cyclops.endpoint; import ch.icclab.cyclops.consume.command.CommandConsumer; -import ch.icclab.cyclops.publish.Messenger; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import com.google.gson.Gson; import org.restlet.representation.Representation; import org.restlet.resource.Post; -import org.restlet.resource.ServerResource; import java.io.IOException; -import java.util.List; /** * Author: Skoviera * Created: 08/07/16 * Description: Handle uploading data frames (the same way as with RabbitMQ) */ -public class CommandEndpoint extends ServerResource { +public class CommandEndpoint extends AbstractEndpoint { - public static String ENDPOINT = "/command"; - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - // logger - final static Logger logger = LogManager.getLogger(CommandEndpoint.class.getName()); + @Override + public String getRoute() { + return "/command"; + } /** * Dispatch and process POST request based on provided parameter @@ -52,7 +43,6 @@ public class CommandEndpoint extends ServerResource { */ @Post public String processPost(Representation entity) throws IOException { - counter.increment(ENDPOINT); try { // first access data consumer @@ -61,20 +51,11 @@ public String processPost(Representation entity) throws IOException { // process the message consumer.consume(entity.getText()); - // get execution status + // get execution status and response CommandConsumer.ExecutionStatus status = consumer.getStatus(); + Object response = consumer.getResponse(); - // was successfully executed - if (status.wasExecuted()) { - - // request restful container - List list = Messenger.getInstance().retrieveRestfulContainer(); - - // return it if it's not empty - return (list != null && !list.isEmpty())? PrettyGson.toJson(list) : status.getMessage(); - } else { - return status.getMessage(); - } + return (status.wasExecuted() && response != null)? new Gson().toJson(response) : status.getMessage(); } catch (Exception e) { return String.format("Error: %s", e.getMessage()); diff --git a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java index 5d48d09..b0605bc 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java @@ -17,37 +17,50 @@ package ch.icclab.cyclops.endpoint; +import ch.icclab.cyclops.consume.command.model.generic.model.Bill; import ch.icclab.cyclops.consume.data.DataConsumer; +import ch.icclab.cyclops.consume.data.DataProcess; +import ch.icclab.cyclops.dto.Measurement; import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.Settings; +import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.load.model.PublisherCredentials; -import ch.icclab.cyclops.util.APICallCounter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import ch.icclab.cyclops.timeseries.InfluxDBClient; +import ch.icclab.cyclops.timeseries.InfluxDBResponse; +import ch.icclab.cyclops.timeseries.QueryBuilder; +import ch.icclab.cyclops.util.loggers.RESTLogger; +import com.google.gson.Gson; +import org.apache.commons.lang.math.NumberUtils; import org.restlet.representation.Representation; +import org.restlet.resource.Get; import org.restlet.resource.Post; -import org.restlet.resource.ServerResource; import java.io.IOException; +import java.util.List; +import java.util.Map; /** * Author: Skoviera * Created: 08/07/16 * Description: Handle uploading data frames (the same way as with RabbitMQ) */ -public class DataEndpoint extends ServerResource { +public class DataEndpoint extends AbstractEndpoint { - public static String ENDPOINT = "/data"; + private static String SELECTED_MEASUREMENT = Bill.class.getSimpleName(); // used for accessing DataConsumer private PublisherCredentials publisher; private String defaultName; - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); + // supported functions + private final static String FUN_COUNT = "count"; - // logger - final static Logger logger = LogManager.getLogger(DataEndpoint.class.getName()); + // supported params + private final static String PARAM_PAGE = "page"; + private final static String PARAM_FROM = "from"; + private final static String PARAM_TO = "to"; + + private final InfluxDBClient dbClient = new InfluxDBClient(); public DataEndpoint() { Settings settings = Loader.getSettings(); @@ -55,6 +68,144 @@ public DataEndpoint() { this.publisher = settings.getPublisherCredentials(); } + @Override + public String getRoute() { + return "/data"; + } + + @Get + public String processGet(){ + + // access query user specified + Map params = getQuery().getValuesMap(); + + // prepare measurement + Measurement measurement = new Measurement(SELECTED_MEASUREMENT, params); + + // has user provided page number? + Integer pageNumber = 0; + if (params.containsKey(PARAM_PAGE)) { + pageNumber = NumberUtils.toInt((String) params.get(PARAM_PAGE), 0); + params.remove(PARAM_PAGE); + } + + // prepare query + QueryBuilder builder = prepareQuery(measurement, params); + // get number of pages + Integer count = getCountForMeasurement(builder); + + // set page limit + Integer pageLimit = Loader.getSettings().getInfluxDBCredentials().getInfluxDBPageSizeLimit(); + measurement.setPageSize(pageLimit); + + // only if it makes sense + if (count > 0) { + // get data from InfluxDB + fillMeasurementWithRecordsFromDatabase(measurement, builder, pageNumber, pageLimit); + measurement.setTotalRecords(count); + } else { + measurement.setDisplayedRecords(0); + measurement.setTotalRecords(0); + } + + // don't forget to log it + RESTLogger.log(String.format("Serving %d records (out of %d) for measurement \"%s\" as page %d", measurement.getDisplayedRecords(), measurement.getTotalRecords(), measurement.getMeasurement(), measurement.getPageNumber())); + + // return measurement + return new Gson().toJson(measurement); + } + + /** + * Prepare query based on measurement and parameters + * @param measurement details + * @param params to be used + * @return QueryBuilder + */ + private QueryBuilder prepareQuery(Measurement measurement, Map params) { + // create query builder + QueryBuilder builder = new QueryBuilder(measurement.getMeasurement()); + + // date range specification FROM + if (params.containsKey(PARAM_FROM)) { + Long from = NumberUtils.toLong(params.get(PARAM_FROM), 0); + + // only if it makes sense + if (from > 0) { + builder.timeFrom(from, DataProcess.TIME_UNIT); + } + + params.remove(PARAM_FROM); + } + + // date range specification TO + if (params.containsKey(PARAM_TO)) { + Long to = NumberUtils.toLong(params.get(PARAM_TO), 0); + + // only if it makes sense + if (to > 0) { + builder.timeTo(to, DataProcess.TIME_UNIT); + } + + params.remove(PARAM_TO); + } + + // add where clauses iteratively + for (Map.Entry entry: params.entrySet()) { + String strNumber = entry.getValue(); + // if it is number + if (NumberUtils.isNumber(strNumber)) { + builder.where(entry.getKey(), NumberUtils.createDouble(strNumber)); + } else { + builder.where(entry.getKey(), entry.getValue()); + } + } + + return builder; + } + + /** + * Count number of records for specified query + * @param builder prepare query builder + * @return Integer + */ + private Integer getCountForMeasurement(QueryBuilder builder) { + try { + // add COUNT select and pass builder + InfluxDBResponse response = dbClient.executeQuery(builder.count(InfluxDBCredentials.COUNTER_FIELD_NAME)); + List countResult = response.getListOfObjects(); + + // get count from first record + return ((Double) countResult.get(0).get(FUN_COUNT)).intValue(); + } catch (Exception ignored) { + return 0; + } + } + + /** + * Add result of database query to measurement + * @param measurement to be filled + * @param builder prepared query + * @param pageNumber to be used + * @param pageLimit for pagination + */ + private void fillMeasurementWithRecordsFromDatabase(Measurement measurement, QueryBuilder builder, Integer pageNumber, Integer pageLimit) { + + try { + // reset selected fields first + builder.resetSelectClause(); + + // add limit and offset, plus execute query + InfluxDBResponse response = dbClient.executeQuery(builder.limit(pageLimit).offset(pageNumber*pageLimit)); + List parsed = response.getListOfObjects(); + + // update measurement + measurement.setData(parsed); + measurement.setPageNumber(pageNumber); + measurement.setDisplayedRecords(parsed.size()); + } catch (Exception ignored) {} + + } + /** * Dispatch and process POST request based on provided parameter * @param entity json @@ -62,7 +213,6 @@ public DataEndpoint() { */ @Post public String processPost(Representation entity) throws IOException { - counter.increment(ENDPOINT); try { // first access data consumer @@ -71,11 +221,7 @@ public String processPost(Representation entity) throws IOException { // process the message consumer.consume(entity.getText()); - // get stats - Long valid = consumer.getNumberOfValidRecords(); - - - return (valid > 0) ? String.format("%d processed", valid) : "Invalid JSON"; + return "Added for processing"; } catch (Exception e) { return String.format("Error: %s", e.getMessage()); diff --git a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java deleted file mode 100644 index 84e6142..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.endpoint; - -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: List available endpoints - */ -public class ListEndpoint extends ServerResource { - - public static String ENDPOINT = "/list"; - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - /** - * This method will return JSON stats for APICallCounter object - * @return JSON - */ - @Get - public String processCommand() { - return PrettyGson.toJson(counter.getAvailableEndpoints()); - } -} \ No newline at end of file diff --git a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java deleted file mode 100644 index 7266745..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java +++ /dev/null @@ -1,194 +0,0 @@ -package ch.icclab.cyclops.endpoint; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.consume.data.DataConsumer; -import ch.icclab.cyclops.dto.Measurement; -import ch.icclab.cyclops.load.Loader; -import ch.icclab.cyclops.load.model.InfluxDBCredentials; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.QueryBuilder; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import ch.icclab.cyclops.util.loggers.RESTLogger; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Endpoint for measurements - */ -public class MeasurementEndpoint extends ServerResource{ - final static Logger logger = LogManager.getLogger(MeasurementEndpoint.class.getName()); - - public static String ENDPOINT = "/measurement"; - public static String ATTRIBUTE = "name"; - - // supported functions - private final static String FUN_COUNT = "count"; - - // supported params - private final static String PARAM_PAGE = "page"; - private final static String PARAM_FROM = "from"; - private final static String PARAM_TO = "to"; - - private final APICallCounter counter = APICallCounter.getInstance(); - private final InfluxDBClient dbClient = InfluxDBClient.getInstance(); - private String attribute; - - /** - * Copy parameters for this particular request - */ - public void doInit() { - attribute = (String) getRequestAttributes().get(ATTRIBUTE); - } - - @Get - public String processGet(){ - counter.increment(ENDPOINT); - - // access query user specified - Map params = getQuery().getValuesMap(); - - // prepare measurement - Measurement measurement = new Measurement(attribute, params); - - // has user provided page number? - Integer pageNumber = 0; - if (params.containsKey(PARAM_PAGE)) { - pageNumber = NumberUtils.toInt((String) params.get(PARAM_PAGE), 0); - params.remove(PARAM_PAGE); - } - - // prepare query - QueryBuilder builder = prepareQuery(measurement, params); - // get number of pages - Integer count = getCountForMeasurement(builder); - - // set page limit - Integer pageLimit = Loader.getSettings().getInfluxDBCredentials().getInfluxDBPageSizeLimit(); - measurement.setPageSize(pageLimit); - - // only if it makes sense - if (count > 0) { - // get data from InfluxDB - fillMeasurementWithRecordsFromDatabase(measurement, builder, pageNumber, pageLimit); - measurement.setTotalRecords(count); - } else { - measurement.setDisplayedRecords(0); - measurement.setTotalRecords(0); - } - - // don't forget to log it - RESTLogger.log(String.format("Serving %d records (out of %d) for measurement \"%s\" as page %d", measurement.getDisplayedRecords(), measurement.getTotalRecords(), measurement.getMeasurement(), measurement.getPageNumber())); - - // return measurement - return PrettyGson.toJson(measurement); - } - - /** - * Prepare query based on measurement and parameters - * @param measurement details - * @param params to be used - * @return QueryBuilder - */ - private QueryBuilder prepareQuery(Measurement measurement, Map params) { - // create query builder - QueryBuilder builder = new QueryBuilder(measurement.getMeasurement()); - - // date range specification FROM - if (params.containsKey(PARAM_FROM)) { - Long from = NumberUtils.toLong(params.get(PARAM_FROM), 0); - - // only if it makes sense - if (from > 0) { - builder.timeFrom(from, DataConsumer.TIME_UNIT); - } - - params.remove(PARAM_FROM); - } - - // date range specification TO - if (params.containsKey(PARAM_TO)) { - Long to = NumberUtils.toLong(params.get(PARAM_TO), 0); - - // only if it makes sense - if (to > 0) { - builder.timeTo(to, DataConsumer.TIME_UNIT); - } - - params.remove(PARAM_TO); - } - - // add where clauses iteratively - for (Map.Entry entry: params.entrySet()) { - String strNumber = entry.getValue(); - // if it is number - if (NumberUtils.isNumber(strNumber)) { - builder.where(entry.getKey(), NumberUtils.createDouble(strNumber)); - } else { - builder.where(entry.getKey(), entry.getValue()); - } - } - - return builder; - } - - /** - * Count number of records for specified query - * @param builder prepare query builder - * @return Integer - */ - private Integer getCountForMeasurement(QueryBuilder builder) { - try { - // add COUNT select and pass builder - List countResult = dbClient.executeQuery(builder.count(InfluxDBCredentials.COUNTER_FIELD_NAME)); - - // get count from first record - return ((Double) countResult.get(0).get(FUN_COUNT)).intValue(); - } catch (Exception ignored) { - return 0; - } - } - - /** - * Add result of database query to measurement - * @param measurement to be filled - * @param builder prepared query - * @param pageNumber to be used - * @param pageLimit for pagination - */ - private void fillMeasurementWithRecordsFromDatabase(Measurement measurement, QueryBuilder builder, Integer pageNumber, Integer pageLimit) { - - // reset selected fields first - builder.resetSelectClause(); - - // add limit and offset, plus execute query - List parsed = dbClient.executeQuery(builder.limit(pageLimit).offset(pageNumber*pageLimit)); - - // update measurement - measurement.setData(parsed); - measurement.setPageNumber(pageNumber); - measurement.setDisplayedRecords(parsed.size()); - } -} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java deleted file mode 100644 index 9ab58c7..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java +++ /dev/null @@ -1,73 +0,0 @@ -package ch.icclab.cyclops.endpoint; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.load.Loader; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.QueryBuilder; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Endpoint for measurements - */ -public class MeasurementsEndpoint extends ServerResource{ - final static Logger logger = LogManager.getLogger(MeasurementsEndpoint.class.getName()); - - public static String ENDPOINT = "/measurements"; - - private final APICallCounter counter = APICallCounter.getInstance(); - private final InfluxDBClient dbClient = InfluxDBClient.getInstance(); - - @Get - public String processGet(){ - counter.increment(ENDPOINT); - - // we will list measurements - QueryBuilder builder = QueryBuilder.getMeasurementsQuery(); - - // get list of measurements - List result = dbClient.executeQuery(builder); - - List list = new ArrayList<>(); - - // iterate over list of maps and add to the list only names - for (Map map: result) { - if (map.containsKey(InfluxDBClient.MEASUREMENT_FIELD_NAME)) { - String name = (String) map.get(InfluxDBClient.MEASUREMENT_FIELD_NAME); - - // we are returning only measurement names that are not the default container - if (!name.equals(Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement())) { - list.add(name); - } - } - } - - // return as JSON - return PrettyGson.toJson(list); - } -} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java index ae6651d..0dc32a4 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java @@ -17,25 +17,22 @@ package ch.icclab.cyclops.endpoint; -import ch.icclab.cyclops.util.APICallCounter; import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; /** * Author: Skoviera * Created: 21/01/16 * Description: Serve application's version over root endpoint */ -public class RootEndpoint extends ServerResource { - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - public static String ENDPOINT = "/"; +public class RootEndpoint extends AbstractEndpoint { @Get public String root(){ - counter.increment(ENDPOINT); - return "RCB Billing micro service - version 1.0.1"; + return "RCB BOX micro service - version 2.1.0"; + } + + @Override + public String getRoute() { + return "/"; } } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java b/core/billing/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java deleted file mode 100644 index ca0cada..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.endpoint; - -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; -import java.util.HashMap; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: This class handles the internal APICallCounter - */ -public class StatusEndpoint extends ServerResource { - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - public static String ENDPOINT = "/status"; - - /** - * This method will return JSON stats for APICallCounter object - * @return JSON - */ - @Get - public String processCommand() { - // first get running statistics - HashMap stats = counter.getRunningStats(); - - // and return - return PrettyGson.toJson(stats); - } -} \ No newline at end of file diff --git a/core/billing/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java b/core/billing/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java new file mode 100644 index 0000000..fb01a40 --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java @@ -0,0 +1,61 @@ +package ch.icclab.cyclops.executor; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Global task executor + */ +public class TaskExecutor { + final static Logger logger = LogManager.getLogger(TaskExecutor.class.getName()); + private static TaskExecutor singleton = new TaskExecutor(); + + private ExecutorService executor; + + private TaskExecutor(){ + executor = obtainSession(); + } + + private ExecutorService obtainSession() { + // create a pool threads (based on available cpu cores) + return Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + } + + public static TaskExecutor getInstance() { return singleton; } + + public void addTask(Runnable task) { + if (executor == null) { + executor = obtainSession(); + } + executor.submit(task); + } + + public void shutDown() { + if (executor != null) { + logger.trace("Shutting down Task Executor"); + executor.shutdownNow(); + executor = null; + } + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/publish/APICaller.java b/core/billing/src/main/java/ch/icclab/cyclops/publish/APICaller.java new file mode 100644 index 0000000..2412449 --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/publish/APICaller.java @@ -0,0 +1,117 @@ +package ch.icclab.cyclops.publish; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.util.BeanList; +import com.google.gson.Gson; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.net.URL; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 29/07/16 + * Description: Call API endpoint + */ +public class APICaller { + + public class Response { + private String object; + private int status; + + public Response(String obj, int stat) { + object = obj; + status = stat; + } + + public int getStatus() { + return status; + } + + public String getAsString() throws Exception { + return object; + } + + public List getAsList() throws Exception { + return new Gson().fromJson(object, List.class); + } + + public Map getAsMap() throws Exception { + return new Gson().fromJson(object, Map.class); + } + + public Object getAsClass(Class clazz) throws Exception { + return new Gson().fromJson(object, clazz); + } + + public List getAsListOfType(Class clazz) throws Exception { + List list = new Gson().fromJson(object, List.class); + return BeanList.populate(list, clazz); + } + } + + /** + * Perform POST query and return Response + * @param endpoint to be called + * @param object to be passed + * @return Response object + * @throws Exception + */ + public APICaller.Response post(URL endpoint, Object object) throws Exception { + // prepare connection + HttpClient client = HttpClientBuilder.create().build(); + + // create request + HttpPost request = new HttpPost(endpoint.toURI()); + StringEntity entity = new StringEntity(new Gson().toJson(object)); + request.addHeader("Accept", "application/json"); + request.addHeader("Content-Type", "application/json"); + request.setEntity(entity); + + // execute response + HttpResponse response = client.execute(request); + return new Response(IOUtils.toString(response.getEntity().getContent()), response.getStatusLine().getStatusCode()); + } + + /** + * Perform GET query and return Response + * @param endpoint to be called + * @return Response object + * @throws Exception + */ + public APICaller.Response get(URL endpoint) throws Exception { + // prepare connection + HttpClient client = HttpClientBuilder.create().build(); + + // create request + HttpGet request = new HttpGet(endpoint.toURI()); + request.addHeader("Accept", "application/json"); + request.addHeader("Content-Type", "application/json"); + + // execute response + HttpResponse response = client.execute(request); + return new Response(IOUtils.toString(response.getEntity().getContent()), response.getStatusLine().getStatusCode()); + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/publish/Messenger.java b/core/billing/src/main/java/ch/icclab/cyclops/publish/Messenger.java index 5d7e9dd..796d5e7 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/publish/Messenger.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/publish/Messenger.java @@ -19,6 +19,7 @@ import ch.icclab.cyclops.util.loggers.DispatchLogger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.List; diff --git a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java index a582d4e..b1850ee 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java @@ -29,6 +29,7 @@ * Description: Batch session for InfluxDB */ public class BatchPointsContainer { + // if you ever run addPoint in parallel stream, make sure points are encapsulated with Collections.synchronizedList() private List points = new ArrayList<>(); public void addPoint(Point.Builder builder) { @@ -37,7 +38,7 @@ public void addPoint(Point.Builder builder) { public BatchPoints getPoints() { // get empty container - BatchPoints container = InfluxDBClient.getInstance().getEmptyContainer(); + BatchPoints container = new InfluxDBClient().getEmptyContainer(); // iterate over points for (Point.Builder builder : points) { @@ -47,4 +48,12 @@ public BatchPoints getPoints() { return container; } + + public Integer size() { + return points.size(); + } + + public Point.Builder getFirstPoint() { + return (size() > 0) ? points.get(0): null; + } } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java new file mode 100644 index 0000000..fb87c61 --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java @@ -0,0 +1,118 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import com.github.wnameless.json.flattener.JsonFlattener; +import com.google.gson.Gson; +import org.influxdb.dto.Point; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Author: Skoviera + * Created: 02/09/16 + * Description: Helper class for DB point generation + */ +public class GenerateDBPoint { + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @return Point.Builder + */ + public static Point.Builder fromObject(Object object) { + return generate(object, null, null, null); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param timeField name + * @param unit of time + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTime(Object object, String timeField, TimeUnit unit) { + return generate(object, timeField, unit, null); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param tags list + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTags(Object object, List tags) { + return generate(object, null, null, tags); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param timeField name + * @param unit of time + * @param tags list + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTimeAndTags(Object object, String timeField, TimeUnit unit, List tags) { + return generate(object, timeField, unit, tags); + } + + /** + * Generate a DBPoint + * @param object to be serialised + * @param time field + * @param unit of time + * @param tags list of tags + * @return Point.Builder or null + */ + private static Point.Builder generate(Object object, String time, TimeUnit unit, List tags) { + try { + // flatten possibly nested object + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(object)); + + // create point builder + Point.Builder builder = Point.measurement(object.getClass().getSimpleName()); + + // if time field was provided + if (time != null && unit != null && flat.containsKey(time)) { + builder.time(TimeStamp.cast(flat.get(time)), unit); + flat.remove(time); + } else { + builder.time(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + } + + // if list of tags was provided + if (tags != null && !tags.isEmpty()) { + for (String tag: tags) { + if (flat.containsKey(tag) && flat.get(tag) instanceof String) { + builder.tag(tag, (String) flat.get(tag)); + flat.remove(tag); + } + } + } + + // finally add individual fields + flat.entrySet().stream().filter(entry -> entry.getValue() != null).forEach(entry -> builder.field(entry.getKey(), entry.getValue())); + + return builder; + } catch (Exception ignored) { + return null; + } + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java index 6aaa4cc..f509425 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java @@ -17,11 +17,13 @@ package ch.icclab.cyclops.timeseries; +import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.util.loggers.TimeSeriesLogger; -import org.apache.commons.beanutils.BeanUtils; +import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.atteo.evo.inflector.English; import org.influxdb.InfluxDB; import org.influxdb.InfluxDBFactory; import org.influxdb.dto.BatchPoints; @@ -29,9 +31,10 @@ import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Author: Skoviera @@ -40,38 +43,23 @@ */ public class InfluxDBClient { final static Logger logger = LogManager.getLogger(InfluxDBClient.class.getName()); - public static String MEASUREMENT_FIELD_NAME = "name"; // singleton - private static InfluxDBClient singleton; private InfluxDBCredentials credentials; - - /** - * Create InfluxDB instance - * @param conf to be used - */ - public static void createInstance(InfluxDBCredentials conf){ - if (singleton == null) { - singleton = new InfluxDBClient(conf); - } - } + private InfluxDB session; /** * Constructor * @param conf to be used */ - private InfluxDBClient(InfluxDBCredentials conf) { + public InfluxDBClient(InfluxDBCredentials conf) { credentials = conf; - - createDatabases(credentials.getInfluxDBTSDB()); + session = obtainSession(); } - /** - * Simple implementation of Singleton class - * @return instance of InfluxDB object - */ - public static InfluxDBClient getInstance() { - return singleton; + public InfluxDBClient() { + credentials = Loader.getSettings().getInfluxDBCredentials(); + session = obtainSession(); } /** @@ -82,6 +70,24 @@ private InfluxDB obtainSession() { return InfluxDBFactory.connect(credentials.getInfluxDBURL(), credentials.getInfluxDBUsername(), credentials.getInfluxDBPassword()); } + /** + * Ping InfluxDB server to see whether it is alive + * @throws Exception + */ + public void ping() throws Exception { + session.ping(); + } + + /** + * Enable batch processing for items that are added as single points + * @param flushPoints flush every X points (for example 2000) + * @param flushFrequency flush every Y time unit (for example 100 ms) + * @param unit time unit (for example milliseconds) + */ + public void configureBatchOnSinglePoints(Integer flushPoints, Integer flushFrequency, TimeUnit unit) { + session.enableBatch(flushPoints, flushFrequency, unit); + } + /** * Save container to InfluxDB * @param container to be persisted @@ -90,9 +96,8 @@ public void persistContainer(BatchPointsContainer container) { persistContainer(container.getPoints()); } private void persistContainer(BatchPoints container) { - InfluxDB con = obtainSession(); TimeSeriesLogger.log(String.format("Saving container with %d points to database", container.getPoints().size())); - con.write(container); + session.write(container); } /** @@ -108,14 +113,11 @@ protected BatchPoints getEmptyContainer() { * @param builder to generate point */ public void persistSinglePoint(Point.Builder builder) { - BatchPoints container = getEmptyContainer(); - // add mandatory hidden field Point point = builder.addField(InfluxDBCredentials.COUNTER_FIELD_NAME, true).build(); - container.point(point); - - persistContainer(container); + // depending on whether batch processing for single points is enabled store immediately or wait for flush + session.write(credentials.getInfluxDBTSDB(), "default", point); } /** @@ -123,41 +125,38 @@ public void persistSinglePoint(Point.Builder builder) { * @param names for database creation */ public void createDatabases(String ... names) { - InfluxDB client = obtainSession(); - // now create required databases for (String name: names) { TimeSeriesLogger.log(String.format("Making sure \"%s\" database exists", name)); - client.createDatabase(name); + session.createDatabase(name); } } /** * Execute query * @param builder QueryBuilder - * @return QueryResult + * @return InfluxDBResponse or null */ - public List executeQuery(QueryBuilder builder) { - InfluxDB client = obtainSession(); + public InfluxDBResponse executeQuery(QueryBuilder builder) { + return executeQuery(Collections.singletonList(builder)); + } + public InfluxDBResponse executeQuery(List builders) { + try { + TimeSeriesLogger.log(String.format("About to execute %d %s", builders.size(), English.plural("query", builders.size()))); - // execute query - QueryResult result = client.query(new Query(builder.build(), credentials.getInfluxDBTSDB())); + // concatenate multiple queries into one + List queries = builders.stream().map(QueryBuilder::build).collect(Collectors.toList()); + String multipleQuery = StringUtils.join(queries, ";"); - // return it parsed as list of maps - return ParseQueryResult.parse(result); - } + // connect to InfluxDB and execute query + QueryResult result = session.query(new Query(multipleQuery, credentials.getInfluxDBTSDB())); - /** - * Execute query and map it to specified class - * @param builder query - * @param clazz for mapping - * @return list or null - */ - public List executeQueryAndMapItToClass(QueryBuilder builder, Class clazz) { - // first execute query and get results - List result = executeQuery(builder); + // return InfluxDB response or null + return (!result.hasError())? new InfluxDBResponse(result): null; - // now return it parsed - return ParseQueryResult.map(result, clazz); + } catch (Exception ignored) { + TimeSeriesLogger.log(String.format("Query execution failed: %s", ignored.getMessage())); + return null; + } } } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java new file mode 100644 index 0000000..f94abac --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java @@ -0,0 +1,136 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.load.model.InfluxDBCredentials; +import ch.icclab.cyclops.util.BeanList; +import com.github.wnameless.json.unflattener.JsonUnflattener; +import com.google.gson.Gson; +import org.influxdb.dto.QueryResult; +import org.joda.time.DateTime; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: InfluxDB response container + */ +public class InfluxDBResponse { + + private static String TIME_FIELD = "time"; + + private QueryResult object; + + public InfluxDBResponse(QueryResult object) { + this.object = object; + } + + /** + * Output underlying InfluxDB structure in JSON + * @return flat table structure + */ + public String getTableAsJson() throws Exception { + return new Gson().toJson(this.object); + + } + + /** + * Parse and transform InfluxDB's underlying structure + * @return JSON + */ + public String getListOfObjectsAsJson() throws Exception { + return new Gson().toJson(getListOfObjects()); + } + + /** + * Parse and transform InfluxDB's underlying structure + * @param clazz definition + * @return List + */ + public List getAsListOfType(Class clazz) throws Exception { + List list = getListOfObjects(); + return BeanList.populate(list, clazz); + } + + /** + * Parse and transform InfluxDB's underlying structure + * @return list of maps + */ + public List getListOfObjects() throws Exception { + + List list = new ArrayList<>(); + + if (object != null && !object.hasError()) { + // iterate over available results + for (QueryResult.Result result: object.getResults()) { + // multiple query responses can have empty results + if (result != null) { + List bulk = result.getSeries(); + if (bulk != null && !bulk.isEmpty()) { + // iterate over available series + for (QueryResult.Series series: result.getSeries()) { + // iterate over individual value sets + Map tags = series.getTags(); + List columns = series.getColumns(); + int time = columns.indexOf(TIME_FIELD); + List> values = series.getValues(); + + // iterate over individual values + if (values != null && !values.isEmpty()) { + for (List value: values) { + // we will store one row here + Map row = new HashMap<>(); + + // add tag values if there are any + if (tags != null && !tags.isEmpty()) { + row.putAll(tags); + } + + // individual entries of the value + int i = 0; + for (Object entry: value) { + // timestamp needs to be in Long and not String + if (time == i) { + entry = new DateTime(entry).getMillis() / 1000; + } + + row.put(columns.get(i), entry); + i++; + } + + // remove property that is used for counting and should never be visible + row.remove(InfluxDBCredentials.COUNTER_FIELD_NAME); + + // return de-flattened map + list.add(new Gson().fromJson(JsonUnflattener.unflatten(new Gson().toJson(row)), Map.class)); + } + } + } + } + } + } + + return list; + } + + throw new Exception("Couldn't parse InfluxDB response"); + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java deleted file mode 100644 index 4ba0769..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java +++ /dev/null @@ -1,128 +0,0 @@ -package ch.icclab.cyclops.timeseries; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.load.model.InfluxDBCredentials; -import com.github.wnameless.json.unflattener.JsonUnflattener; -import com.google.gson.Gson; -import org.apache.commons.beanutils.BeanUtils; -import org.influxdb.dto.QueryResult; -import org.joda.time.DateTime; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Parse InfluxDB's Query Result - */ -public class ParseQueryResult { - - private static String TIME_FIELD = "time"; - - public static List parse(QueryResult data) { - // container for result - List container = new ArrayList<>(); - - // mapping - Gson gson = new Gson(); - - try { - // go over all results - for (QueryResult.Result result : data.getResults()) { - - // go over individual series - for (QueryResult.Series serie : result.getSeries()) { - - // store columns here - List columns = new ArrayList<>(); - columns.addAll(serie.getColumns()); - - // get time field position - int time = columns.indexOf(TIME_FIELD); - - // iterate over values - for (List values : serie.getValues()) { - // we will store one row here - Map row = new HashMap<>(); - - // add tag values if there are any - Map tags = serie.getTags(); - if (tags != null && !tags.isEmpty()) { - row.putAll(tags); - } - - // access objects - int i = 0; - for (Object value: values) { - - // timestamp needs to be in Long and not String - if (time == i) { - try { - // return seconds not milliseconds - value = new DateTime(value).getMillis() / 1000; - } catch (Exception ignored) { - } - } - - row.put(columns.get(i), value); - i++; - } - - // delete meta field - row.remove(InfluxDBCredentials.COUNTER_FIELD_NAME); - - // de-flatten structure - String rich = JsonUnflattener.unflatten(gson.toJson(row)); - container.add(gson.fromJson(rich, Map.class)); - } - } - } - } catch (Exception ignored) { - // in case of empty QueryResult body do nothing - } - - return container; - } - - public static List map(List list, Class clazz) { - List mapped = new ArrayList<>(); - - // iterate and map those objects - if (list != null) { - for (Map map : list) { - try { - Object bean = clazz.newInstance(); - - // map HashMap to POJO - BeanUtils.populate(bean, map); - - // add it to list of mapped CDRs - mapped.add(bean); - - } catch (Exception ignored) { - return null; - } - } - } - - return mapped; - } -} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/PrettyGson.java b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java similarity index 61% rename from core/rc/cdr/src/main/java/ch/icclab/cyclops/util/PrettyGson.java rename to core/billing/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java index d54d16a..f64855f 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/PrettyGson.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java @@ -1,4 +1,4 @@ -package ch.icclab.cyclops.util; +package ch.icclab.cyclops.timeseries; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -16,21 +16,22 @@ * under the License. */ -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import java.util.concurrent.TimeUnit; /** * Author: Skoviera - * Created: 08/03/16 - * Description: Always use pretty JSON output + * Created: 31/08/16 + * Description: Shared InfluxDB session */ -public class PrettyGson { +public class SharedInfluxDBSession { + private static InfluxDBClient session; - private static Gson getGson() { - return new GsonBuilder().setPrettyPrinting().create(); - } + public static InfluxDBClient getSession() { + if (session == null) { + session = new InfluxDBClient(); + session.configureBatchOnSinglePoints(2000, 100, TimeUnit.MILLISECONDS); + } - public static String toJson(Object object) { - return getGson().toJson(object); + return session; } } diff --git a/core/billing/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java new file mode 100644 index 0000000..f77a4db --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java @@ -0,0 +1,53 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import java.math.BigDecimal; + +/** + * Author: Skoviera + * Created: 06/09/16 + * Description: Various utils for TimeStamp object manipulation + */ +public class TimeStamp { + + /** + * Parse long from unknown type + * @param unknown object + * @return Long or null + */ + public static Long cast(Object unknown) { + try { + BigDecimal dec = (BigDecimal) unknown; + return dec.longValue(); + } catch (Exception a) { + try { + return Long.parseLong((String) unknown); + } catch (Exception b) { + try { + return (Long) unknown; + } catch (Exception c) { + try { + return Long.valueOf(unknown.toString()); + } catch (Exception d) { + return null; + } + } + } + } + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/util/APICallCounter.java b/core/billing/src/main/java/ch/icclab/cyclops/util/APICallCounter.java deleted file mode 100644 index ff640c1..0000000 --- a/core/billing/src/main/java/ch/icclab/cyclops/util/APICallCounter.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.util; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: Class that will count API Calls and then provide the information (and reset counters) - */ -public class APICallCounter { - // we need a singleton for this - private static APICallCounter singleton = new APICallCounter(); - - // Hash-map to be used for counting - private HashMap map; - - // list used for holding them all - private ArrayList list; - - // Hash-map used as a template (for registering fields) - private HashMap template; - - /** - * Private Constructor is necessary for the singleton model - */ - private APICallCounter() { - template = new HashMap<>(); - list = new ArrayList<>(); - map = new HashMap<>(); - } - - /** - * Simply return singleton instance - * @return APICallCounter instance - */ - public static APICallCounter getInstance() { - return singleton; - } - - /** - * Will increment the counter for provided key - * In case that this key was not previously registered, it will add it to the hash-map - * However on dump, it will get removed and only the registered ones will stay - * @param key meaning API endpoint - */ - public void increment(String key) { - Object value = map.get(key); - - // now save new value or increment it - if (value == null) { - map.put(key, 1); - } else { - map.put(key, ((Integer) value) + 1); - } - } - - /** - * Register an API endpoint to the counter - * @param key as endpoint string - */ - public void registerEndpoint(String key) { - template.put(key, 0); - map.put(key, 0); - list.add(key); - } - - /** - * Register an API endpoint for listing - * @param key as endpoint - */ - public void registerEndpointWithoutCounting(String key) { - list.add(key); - } - - /** - * Will ask for running stats and DUMP the old hash-map so it can start from zero again - * @return hash-map - */ - public HashMap getRunningStats() { - // make a deep copy - HashMap stats = new HashMap(); - stats.putAll(map); - - // reset map based on registered entries in template - map = new HashMap(); - map.putAll(template); - - // return deep copy - return stats; - } - - /** - * Return list of available endpoints - * @return list - */ - public List getAvailableEndpoints() { - return list; - } -} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/util/BeanList.java b/core/billing/src/main/java/ch/icclab/cyclops/util/BeanList.java new file mode 100644 index 0000000..6c9d265 --- /dev/null +++ b/core/billing/src/main/java/ch/icclab/cyclops/util/BeanList.java @@ -0,0 +1,54 @@ +package ch.icclab.cyclops.util; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.beanutils.BeanUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Utility to populate list of beans + */ +public class BeanList { + public static List populate(List list, Class clazz) { + List mapped = new ArrayList<>(); + + // iterate and map those objects + if (list != null) { + for (Map map : list) { + try { + Object bean = clazz.newInstance(); + + // map HashMap to POJO + BeanUtils.populate(bean, map); + + // add it to list of mapped CDRs + mapped.add(bean); + + } catch (Exception ignored) { + return null; + } + } + } + + return mapped; + } +} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java b/core/billing/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java index 2aa8212..7c61a5e 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java +++ b/core/billing/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java @@ -17,6 +17,7 @@ */ import ch.icclab.cyclops.consume.RabbitMQListener; +import ch.icclab.cyclops.executor.TaskExecutor; import ch.icclab.cyclops.publish.RabbitMQPublisher; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,7 +33,10 @@ public class ShutDownListener extends Thread{ @Override public void run() { - logger.trace("We are shutting down Billing micro service"); + logger.trace("We are shutting down BOX micro service"); + + // shut down executor service + TaskExecutor.getInstance().shutDown(); // and Consumer (RabbitMQ) RabbitMQListener.shutDown(); diff --git a/core/billing/src/main/resources/log4j2.component.properties b/core/billing/src/main/resources/log4j2.component.properties new file mode 100644 index 0000000..683a3da --- /dev/null +++ b/core/billing/src/main/resources/log4j2.component.properties @@ -0,0 +1 @@ +Log4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector \ No newline at end of file diff --git a/core/billing/src/main/resources/log4j2.xml b/core/billing/src/main/resources/log4j2.xml index 550fefd..725ca64 100644 --- a/core/billing/src/main/resources/log4j2.xml +++ b/core/billing/src/main/resources/log4j2.xml @@ -1,27 +1,27 @@ - - - - - - - + + + + + + + - - + + - - + + - - + + - - + + - + diff --git a/core/rc/cdr/ReleaseNotes.txt b/core/rc/cdr/ReleaseNotes.txt new file mode 100644 index 0000000..9fda046 --- /dev/null +++ b/core/rc/cdr/ReleaseNotes.txt @@ -0,0 +1,7 @@ +Version: 2.1.0 +Date: 16/September/2016 +Notes: Data ingestion and query execution optimisations, new CDR output format (part of FlushCDR command) + +Version: 2.0.0 +Date: 1/July/2016 +Notes: Initial release of the RCB Cyclops CDR micro service \ No newline at end of file diff --git a/core/rc/cdr/bin/cdr.jar b/core/rc/cdr/bin/cdr.jar index 2d0815f..ad42d6b 100644 Binary files a/core/rc/cdr/bin/cdr.jar and b/core/rc/cdr/bin/cdr.jar differ diff --git a/core/rc/cdr/config/cdr.conf b/core/rc/cdr/config/cdr.conf index 41c4144..e0a7854 100644 --- a/core/rc/cdr/config/cdr.conf +++ b/core/rc/cdr/config/cdr.conf @@ -28,4 +28,4 @@ ConsumerPassword=guest ConsumerPort=5672 ConsumerVirtualHost=/ ConsumerDataQueue=cyclops.cdr.consume -ConsumerCommandsQueue=cyclops.cdr.commands \ No newline at end of file +ConsumerCommandsQueue=cyclops.cdr.commands diff --git a/core/rc/cdr/pom.xml b/core/rc/cdr/pom.xml index 7fb55c2..50c0574 100644 --- a/core/rc/cdr/pom.xml +++ b/core/rc/cdr/pom.xml @@ -7,19 +7,20 @@ ch.icclab.cyclops.cdr cyclops-cdr jar - 0.0.1 + 2.1.0 CDR 1.8 UTF-8 UTF-8 - 2.3.4 - 2.9.3 - 2.5 - 3.6.1 - 1.7.12 + 2.3.7 + 2.9.4 + 3.1.0 + 3.6.5 + 1.7.21 2.2 + 2.6.2 @@ -103,7 +104,7 @@ javax.servlet - servlet-api + javax.servlet-api ${servlet-version} @@ -119,47 +120,47 @@ org.slf4j jul-to-slf4j - 1.7.12 + ${slf4j-version} runtime org.apache.logging.log4j log4j-api - 2.3 + ${log4j-version} org.apache.logging.log4j log4j-core - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-web - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-jul - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-slf4j-impl - 2.3 + ${log4j-version} runtime org.json json - 20140107 + 20160212 com.google.code.gson gson - 2.6.2 + 2.7 io.gsonfire @@ -179,17 +180,32 @@ com.metapossum metapossum-scanner - 1.0 + 1.0.1 com.github.wnameless json-flattener - 0.1.6 + 0.2.2 commons-beanutils commons-beanutils 1.9.2 + + org.apache.httpcomponents + httpclient + 4.5.2 + + + com.lmax + disruptor + 3.3.5 + + + org.atteo + evo-inflector + 1.2.1 + \ No newline at end of file diff --git a/core/rc/cdr/scripts/compile.sh b/core/rc/cdr/scripts/compile.sh index 69ae16e..77f7f0a 100755 --- a/core/rc/cdr/scripts/compile.sh +++ b/core/rc/cdr/scripts/compile.sh @@ -26,4 +26,4 @@ mvn dependency:tree mvn package assembly:single cd target -mv cyclops-cdr-0.0.1-jar-with-dependencies.jar ../bin/cdr.jar +mv cyclops-cdr-2.1.0-jar-with-dependencies.jar ../bin/cdr.jar diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/application/Main.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/application/Main.java index 8266c2f..8ca12da 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/application/Main.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/application/Main.java @@ -21,6 +21,7 @@ import ch.icclab.cyclops.consume.data.DataConsumer; import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.Settings; +import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.load.model.PublisherCredentials; import ch.icclab.cyclops.publish.RabbitMQPublisher; import ch.icclab.cyclops.timeseries.InfluxDBClient; @@ -38,6 +39,11 @@ */ public class Main extends Application{ + static { + // Nothing can appear before this initializer + System.setProperty("Log4jContextSelector", "org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"); + } + final static Logger logger = LogManager.getLogger(Main.class.getName()); private static final int OK_HELP = 1; @@ -250,8 +256,13 @@ private static void checkCustomPortOption(String[] args) { */ private static void checkAndConfigureInfluxDB() { try { - logger.trace("Binding to InfluxDB"); - InfluxDBClient.createInstance(Loader.getSettings().getInfluxDBCredentials()); + logger.trace("Binding to InfluxDB and creating databases"); + InfluxDBCredentials credentials = Loader.getSettings().getInfluxDBCredentials(); + InfluxDBClient client = new InfluxDBClient(credentials); + + client.ping(); + client.createDatabases(credentials.getInfluxDBTSDB()); + } catch (Exception e) { String log = String.format("Couldn't connect to InfluxDb: %s", e.getMessage()); logger.error(log); diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/application/Service.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/application/Service.java index d456639..cdbc700 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/application/Service.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/application/Service.java @@ -18,12 +18,16 @@ package ch.icclab.cyclops.application; import ch.icclab.cyclops.endpoint.*; -import ch.icclab.cyclops.util.APICallCounter; +import ch.icclab.cyclops.util.RegexParser; +import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.restlet.Restlet; import org.restlet.routing.Router; +import java.util.ArrayList; +import java.util.List; + /** * Author: Skoviera * Created: 21/01/16 @@ -36,63 +40,46 @@ public class Service { // Router for registering api endpoints private Router router; - // API counter - private APICallCounter counter; - /** - * Attaching endpoints to router + * This method handles the incoming request and routes it to the appropriate resource class */ - private void attachTheRest() { - logger.trace("Attaching measurement endpoint"); - router.attach(String.format("%s/{%s}", MeasurementEndpoint.ENDPOINT, MeasurementEndpoint.ATTRIBUTE), MeasurementEndpoint.class); - counter.registerEndpoint(MeasurementEndpoint.ENDPOINT); - - router.attach(MeasurementsEndpoint.ENDPOINT, MeasurementsEndpoint.class); - counter.registerEndpoint(MeasurementsEndpoint.ENDPOINT); + public Restlet createInboundRoot() throws Exception { - logger.trace("Attaching data endpoint"); - router.attach(DataEndpoint.ENDPOINT, DataEndpoint.class); - counter.registerEndpoint(DataEndpoint.ENDPOINT); + logger.trace("Initialising CDR microservice and creating routes"); - logger.trace("Attaching command endpoint"); - router.attach(CommandEndpoint.ENDPOINT, CommandEndpoint.class); - counter.registerEndpoint(CommandEndpoint.ENDPOINT); - } + router = attachRoutes(); - /** - * Construct application by accessing context, creating router and counter - */ - private void initialiseApplication() { - logger.trace("Initialising CDR microservice"); + logger.trace("Routes for CDR microservice successfully created"); - router = new Router(); - counter = APICallCounter.getInstance(); + return router; } /** - * This method handles the incoming request and routes it to the appropriate resource class + * Attach routes that extend AbstractEndpoint class + * @return router */ - public Restlet createInboundRoot() throws Exception { - - // let's start by initialising and loading configuration settings - initialiseApplication(); - - logger.trace("Creating routes for CDR microservice"); - - // root, status and list endpoints - router.attach(RootEndpoint.ENDPOINT, RootEndpoint.class); - counter.registerEndpoint(RootEndpoint.ENDPOINT); - - router.attach(StatusEndpoint.ENDPOINT, StatusEndpoint.class); - counter.registerEndpointWithoutCounting(StatusEndpoint.ENDPOINT); - - router.attach(ListEndpoint.ENDPOINT, ListEndpoint.class); - counter.registerEndpointWithoutCounting(ListEndpoint.ENDPOINT); - - // attach other endpoints - attachTheRest(); - - logger.trace("Routes for CDR microservice successfully created"); + private Router attachRoutes() { + Router router = new Router(); + + List list = new ArrayList<>(); + + try { + // find all endpoints and add them to the list + list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) -> !AbstractEndpoint.class.getSimpleName(). + equals(RegexParser.getFileName(fileName))).scan(AbstractEndpoint.class.getPackage().getName())); + } catch (Exception ignored) {} + + // create routes + for (Class clazz : list) { + // only those which extend Endpoint + if (AbstractEndpoint.class.isAssignableFrom(clazz)) { + try { + // get endpoint's route path + String route = ((AbstractEndpoint) clazz.newInstance()).getRoute(); + router.attach(route, clazz); + } catch (Exception ignored) {} + } + } return router; } diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/Command.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/Command.java index 5419bb0..6616256 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/Command.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/Command.java @@ -27,8 +27,9 @@ public abstract class Command { /** * Every command has to implement execute method + * @return Object of your selection or Null */ - protected abstract void execute(); + protected abstract Object execute(); //===== Getters and Setters public String get_class() { diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java index f3f5b83..3c01713 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java @@ -10,6 +10,8 @@ */ public class CommandConsumer extends AbstractConsumer { + private Object response; + private ExecutionStatus status; public class ExecutionStatus { private Boolean executed; @@ -42,7 +44,7 @@ public void consume(String content) { if (command != null) { try { - command.execute(); + response = command.execute(); status = new ExecutionStatus(true, String.format("[OK] command \"%s\" successfully executed", command.get_class())); CommandLogger.log(status.getMessage()); @@ -59,4 +61,7 @@ public void consume(String content) { public ExecutionStatus getStatus() { return status; } + public Object getResponse() { + return response; + } } diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java index 66ed0e2..2070dd0 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java @@ -24,6 +24,7 @@ import io.gsonfire.TypeSelector; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.List; @@ -37,25 +38,25 @@ public class CommandMapping { final static Logger logger = LogManager.getLogger(CommandMapping.class.getName()); private static final Gson gson = new GsonFireBuilder() - .registerTypeSelector(Command.class, new TypeSelector() { - @Override - public Class getClassForElement(JsonElement jsonElement) { - try { + .registerTypeSelector(Command.class, new TypeSelector() { + @Override + public Class getClassForElement(JsonElement jsonElement) { + try { - String clazz = jsonElement.getAsJsonObject().get(Command.FIELD_FOR_MAPPING).getAsString(); + String clazz = jsonElement.getAsJsonObject().get(Command.FIELD_FOR_MAPPING).getAsString(); - // recursively find correct classes - List list = new ArrayList<>(); - list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) - -> clazz.equals(RegexParser.getFileName(fileName))).scan(CommandMapping.class.getPackage().getName())); + // recursively find correct classes + List list = new ArrayList<>(); + list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) + -> clazz.equals(RegexParser.getFileName(fileName))).scan(CommandMapping.class.getPackage().getName())); - // and use the first one - return (Class) Class.forName(list.get(0).getName()); - } catch (Exception e) { - return null; + // and use the first one + return (Class) Class.forName(list.get(0).getName()); + } catch (Exception e) { + return null; + } } - } - }).createGson(); + }).createGson(); /** * Map object to provided class diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/ExampleCommand.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/ExampleCommand.java deleted file mode 100644 index ea7040e..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/ExampleCommand.java +++ /dev/null @@ -1,45 +0,0 @@ -package ch.icclab.cyclops.consume.command.model; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.consume.command.Command; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * Author: Skoviera - * Created: 29/04/16 - * Description: Example command - */ -public class ExampleCommand extends Command { - final static Logger logger = LogManager.getLogger(ExampleCommand.class.getName()); - - private String parameter; - - @Override - protected void execute() { - // do something - } - - //====== Getters and Setters - public String getParameter() { - return parameter; - } - public void setParameter(String parameter) { - this.parameter = parameter; - } -} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/FlushData.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/FlushData.java new file mode 100644 index 0000000..da1f598 --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/FlushData.java @@ -0,0 +1,157 @@ +package ch.icclab.cyclops.consume.command.model.generic; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.consume.command.Command; +import ch.icclab.cyclops.consume.command.model.generic.model.CDR; +import ch.icclab.cyclops.publish.APICaller; +import ch.icclab.cyclops.timeseries.InfluxDBClient; +import ch.icclab.cyclops.timeseries.InfluxDBResponse; +import ch.icclab.cyclops.timeseries.QueryBuilder; +import ch.icclab.cyclops.util.loggers.CommandLogger; + +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * Author: Skoviera + * Created: 09/09/16 + * Description: Flush data from database + */ +public class FlushData extends Command{ + + // TODO add service discovery + private static String URL = "localhost:4571"; + + // mandatory + private Long from; + private Long to; + + // by default we are synchronously pushing to the next step + private Boolean sync; + + // in case we want data set be returned + private Boolean output; + + // optional + private List accounts; + + public static class Measurement { + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + @Override + protected Object execute() { + try { + // sanity checks first + if (from == null || to == null || from < 0l || to <= from) { + return "[ERROR] invalid FROM and TO"; + } + + // get list of measurements + List measurements = getListOfMeasurements(); + if (measurements != null && !measurements.isEmpty()) { + List list = new ArrayList<>(); + + // query all measurements and get Usage Data + measurements.stream().forEach(measurement -> list.addAll(queryMeasurement(measurement))); + + // we got some results from database + if (!list.isEmpty()) { + + // do we want to push it to the next micro service or not + if (sync == null || sync) { + CommandLogger.log(String.format("Flushing data (%d items) to Billing Rule engine (%s)", list.size(), URL)); + + new APICaller().post(new URL(String.format("http://%s/ruleengine/facts", URL)), list); + } + + // should we simply return the data set? + return (output != null && output)? list: String.format("Flushed %d items", list.size()); + } else { + CommandLogger.log("No data to be pushed into Billing Rule engine, as list of records received from underlying database is zero"); + return "No data"; + } + } else { + return "No data"; + } + + } catch (Exception e) { + // something required was not provided, do nothing + return String.format("[ERROR] %s", e.getMessage()); + } + } + + /** + * Get list of measurements + * @return list of null + */ + private List getListOfMeasurements(){ + try { + // prepare and execute query + QueryBuilder builder = QueryBuilder.getMeasurementsQuery(); + InfluxDBResponse result = new InfluxDBClient().executeQuery(builder); + + // map response to Measurement class + List measurements = result.getAsListOfType(Measurement.class); + + // get it as a list of Strings and filter out UDR measurement + return measurements.stream().map(Measurement::getName).collect(Collectors.toList()); + } catch (Exception e) { + return null; + } + } + + /** + * Query database and list usage data for specified measurement + * @param measurement to query + * @return CDR list + */ + private List queryMeasurement(String measurement) { + try { + + List queries = new ArrayList<>(); + + // if there are multiple accounts + if (accounts != null && !accounts.isEmpty()) { + for (String account: accounts) { + queries.add(new QueryBuilder(measurement).timeFrom(from, TimeUnit.SECONDS).timeTo(to, TimeUnit.SECONDS).where("account", account)); + } + } else { + queries.add(new QueryBuilder(measurement).timeFrom(from, TimeUnit.SECONDS).timeTo(to, TimeUnit.SECONDS)); + } + + InfluxDBResponse response = new InfluxDBClient().executeQuery(queries); + + // parse as list of UsageData + return response.getAsListOfType(CDR.class); + } catch (Exception ignored) { + return new ArrayList<>(); + } + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/CDR.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/CDR.java new file mode 100644 index 0000000..5fd3efd --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/CDR.java @@ -0,0 +1,83 @@ +package ch.icclab.cyclops.consume.command.model.generic.model; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import java.util.List; + +/** + * Author: Skoviera + * Created: 06/09/16 + * Description: CDR output + */ +public class CDR { + private String _class = getClass().getSimpleName(); + private String account; + private Long time; + private Long from; + private Long to; + private Double charge; + private List data; + + public String get_class() { + return _class; + } + public void set_class(String _class) { + this._class = _class; + } + + public String getAccount() { + return account; + } + public void setAccount(String account) { + this.account = account; + } + + public Long getTime() { + return time; + } + public void setTime(Long time) { + this.time = time; + } + + public Long getFrom() { + return from; + } + public void setFrom(Long from) { + this.from = from; + } + + public Long getTo() { + return to; + } + public void setTo(Long to) { + this.to = to; + } + + public Double getCharge() { + return charge; + } + public void setCharge(Double charge) { + this.charge = charge; + } + + public List getData() { + return data; + } + public void setData(List data) { + this.data = data; + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java index 4bb2e74..584efd3 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java @@ -1,24 +1,10 @@ package ch.icclab.cyclops.consume.data; import ch.icclab.cyclops.consume.AbstractConsumer; -import ch.icclab.cyclops.load.Loader; +import ch.icclab.cyclops.executor.TaskExecutor; import ch.icclab.cyclops.load.model.PublisherCredentials; -import ch.icclab.cyclops.publish.Messenger; -import ch.icclab.cyclops.timeseries.BatchPointsContainer; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.RemoveNullValues; -import ch.icclab.cyclops.util.RegexParser; -import ch.icclab.cyclops.util.loggers.DataLogger; -import com.github.wnameless.json.flattener.JsonFlattener; -import com.google.gson.Gson; -import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.influxdb.dto.Point; - -import java.math.BigDecimal; -import java.util.*; -import java.util.concurrent.TimeUnit; /** * Author: Skoviera @@ -28,371 +14,19 @@ public class DataConsumer extends AbstractConsumer { final static Logger logger = LogManager.getLogger(DataConsumer.class.getName()); - private static InfluxDBClient influxDBClient = InfluxDBClient.getInstance(); - private static Messenger messenger = Messenger.getInstance(); - - public static TimeUnit TIME_UNIT = TimeUnit.SECONDS; - private PublisherCredentials publisherSettings; private String defaultMeasurementName; + private TaskExecutor executor; - private Long validRecords; - - private BatchPointsContainer container; - private List broadcast; - private Map> dispatch; public DataConsumer(String name, PublisherCredentials settings) { this.defaultMeasurementName = name; this.publisherSettings = settings; + this.executor = TaskExecutor.getInstance(); } @Override public void consume(String content) { - initialise(); - - try { - // try to map it as array - List array = new Gson().fromJson(content, List.class); - - // iterate over object entries - array.forEach(this::processDataFrame); - - } catch (Exception ignored) { - // this means it was not an array to begin with, just a simple object - processDataFrame(content); - } - - finalise(); - } - - public Long getNumberOfValidRecords() { - return validRecords; - } - - private void initialise() { - validRecords = 0l; - container = new BatchPointsContainer(); - broadcast = new ArrayList<>(); - dispatch = new HashMap<>(); - } - - private void finalise() { - // persist points that were created - influxDBClient.persistContainer(container); - - // broadcast - if (broadcast != null && !broadcast.isEmpty()) { - if (broadcast.size() == 1) { - // broadcast it as value - messenger.broadcast(broadcast.get(0)); - } else { - // broadcast it as array - messenger.broadcast(broadcast); - } - } - - // dispatch - if (dispatch != null && !dispatch.isEmpty()) { - for (Map.Entry entry: dispatch.entrySet()) { - List list = (List) entry.getValue(); - - if (list.size() == 1) { - // publish it as value - messenger.publish(list.get(0), (String) entry.getKey()); - } else { - // publish it as array - messenger.publish(list, (String) entry.getKey()); - } - } - } - } - - /** - * Process data frame as OBJECT - * @param obj to be processed - */ - private void processDataFrame(Map obj) { - storeAndPublish(obj); - } - - /** - * Process data frame as STRING - * @param str to be processed - */ - private void processDataFrame(String str) { - try { - Map obj = new Gson().fromJson(str, Map.class); - - storeAndPublish(obj); - } catch (Exception ignored) { - // if incoming JSON is not valid we are simply skipping it - } - } - - /** - * Process, store and publish incoming object - * @param obj representation - */ - private void storeAndPublish(Map obj) { - try { - // now parse it and create a database Point - ConsumedData data = processDataAndGetPoint(obj); - - // and finally persist it - container.addPoint(data.getPoint()); - - // publish or broadcast if desired - if (data.shouldPublish()) { - - // reapply class definition - obj.put(DataMapping.FIELD_FOR_MAPPING, data.getClazz()); - - // where to publish - if (data.doNotBroadcastButRoute()) { - String clazz = data.getClazz(); - - // add to internal dispatch structure - if (dispatch.containsKey(clazz)) { - // we need to fetch the list - List list = dispatch.get(clazz); - list.add(obj); - } else { - // we need to create a new one - List list = new ArrayList<>(); - list.add(obj); - - dispatch.put(clazz, list); - } - - } else { - broadcast.add(obj); - } - } - - validRecords += 1; - - } catch (Exception notValidJson) { - DataLogger.log(String.format("Received event/measurement is not a valid JSON: %s", notValidJson.getMessage())); - } - } - - /** - * Data holder for consumed data including database point and mapping guidelines - */ - private class ConsumedData { - private String clazz; - private Point.Builder builder; - private Boolean shouldPublish; - private Boolean doNotBroadcastButRoute; - - public ConsumedData(String clazz, Point.Builder builder, Boolean shouldPublish, Boolean doNotBroadcastButRoute) { - this.clazz = clazz; - this.builder = builder; - this.shouldPublish = shouldPublish; - this.doNotBroadcastButRoute = doNotBroadcastButRoute; - } - - public ConsumedData(String clazz, Point.Builder builder, DataMapping guideline) { - this.clazz = clazz; - this.builder = builder; - this.shouldPublish = guideline.shouldPublish(); - this.doNotBroadcastButRoute = guideline.doNotBroadcastButRoute(); - } - - public String getClazz() { - return clazz; - } - public Point.Builder getPoint() { - return builder; - } - public Boolean shouldPublish() { - return shouldPublish; - } - public Boolean doNotBroadcastButRoute() { - return doNotBroadcastButRoute; - } - } - - /** - * Process and create database Point - * @param map to be processed - * @return point - */ - private ConsumedData processDataAndGetPoint(Map map) { - try { - // check whether type field is present - String clazz = (String) map.get(DataMapping.FIELD_FOR_MAPPING); - - // find corresponding classes - Set set = new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) - -> clazz.equals(RegexParser.getFileName(fileName))).scan(DataConsumer.class.getPackage().getName()); - - // move to the first element - Optional first = set.stream().findFirst(); - - if (first.isPresent()) { - // access object based on provided class definition - DataMapping guideline = (DataMapping) Class.forName(first.get().getName()).newInstance(); - - DataLogger.log(String.format("Event/measurement received with \"%s\" guidelines", clazz)); - - // by default we will work with the original map - Map processed = map; - try { - // let's call user specified pre-processing method - Map preProcessing = guideline.preProcess(map); - - // assign new one if it's valid - if (preProcessing != null && !preProcessing.isEmpty()) { - processed = preProcessing; - } - } catch (Exception ignored) { - } - - // we will be storing flattened version of the map - Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(processed)); - - // remove type from original HashMap - flat.remove(DataMapping.FIELD_FOR_MAPPING); - - // extract tags from flattened json - Map tags = extractTags(flat, guideline.getTagNames()); - - // parse time field if it was provided - String timeField = guideline.getTimeField(); - Long timeStamp = (timeField != null && flat.containsKey(timeField))? getTimeStamp(flat.get(timeField)): null; - if (timeStamp != null && timeStamp >= 0) flat.remove(timeField); - - // determine correct time unit - TimeUnit unit = guideline.getTimeUnit(); - if (unit == null) { - unit = TIME_UNIT; - } - // finally construct database point - Point.Builder builder = constructPoint(clazz, timeStamp, unit, flat, tags); - return new ConsumedData(clazz, builder, guideline); - } else { - // guideline was not present or wasn't valid - throw new Exception(); - } - - } catch (Exception withoutGuidelines){ - // mapping failed, guideline was not provided, use default - DataLogger.log("Event/measurement received without valid guidelines"); - - Point.Builder builder; - - Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(map)); - - // even though we don't have mapping template, we still want to have proper measurement name if TYPE is present - if (flat.containsKey(DataMapping.FIELD_FOR_MAPPING)){ - - // get that name - String measurement = (String) flat.get(DataMapping.FIELD_FOR_MAPPING); - - // remove it from flat - flat.remove(DataMapping.FIELD_FOR_MAPPING); - - // construct point - builder = constructPoint(measurement, flat); - } else { - builder = constructPoint(defaultMeasurementName, flat); - } - - // determine publisher preferences - Boolean shouldPublish = publisherSettings.getPublisherIncludeAlsoUnknown(); - Boolean doNotRouteButBroadcast = publisherSettings.getPublisherByDefaultDispatchInsteadOfBroadcast(); - String clazz = Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement(); - - return new ConsumedData(clazz, builder, shouldPublish, doNotRouteButBroadcast); - } - } - - /** - * Construct database Point - * @param measurementName to be used - * @param fields as map - * @return point builder - */ - private Point.Builder constructPoint(String measurementName, Map fields) { - return Point.measurement(measurementName).fields(RemoveNullValues.fromMap(fields)); - - } - - /** - * Construct database Point - * @param measurementName to be used - * @param timeStamp to be marked - * @param unit of time to be used - * @param fields as map - * @param tags as map - * @return point builder - */ - private Point.Builder constructPoint(String measurementName, Long timeStamp, TimeUnit unit, Map fields, Map tags) { - Point.Builder builder = Point.measurement(measurementName); - - // apply time stamp with time unit - if (timeStamp != null && timeStamp >= 0 && unit != null) { - builder.time(timeStamp, unit); - } - - // apply specified tags - if (tags != null && !tags.isEmpty()) { - builder.tag(RemoveNullValues.fromMap(tags)); - } - - // and finally add fields - builder.fields(RemoveNullValues.fromMap(fields)); - - return builder; - } - - /** - * Move tags from content into variable based on provided list of tags - * @param content to be extracted - * @param tagList definition - * @return extracted tags or null - */ - private Map extractTags(Map content, List tagList) { - if (tagList != null) { - Map tags = new HashMap<>(); - - // iterate over list of tags and move them from the original content - for (String tag: tagList) { - tags.put(tag, (String) content.get(tag)); - content.remove(tag); - } - - return tags; - } else { - return null; - } - } - - /** - * Parse long from unknown type - * @param unknown object - * @return Long or null - */ - private Long getTimeStamp(Object unknown) { - try { - BigDecimal dec = (BigDecimal) unknown; - return dec.longValue(); - } catch (Exception a) { - try { - return Long.parseLong((String) unknown); - } catch (Exception b) { - try { - return (Long) unknown; - } catch (Exception c) { - try { - return Long.valueOf(unknown.toString()); - } catch (Exception d) { - return null; - } - } - } - } + executor.addTask(new DataProcess(content, publisherSettings, defaultMeasurementName)); } } diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java new file mode 100644 index 0000000..25bd5c0 --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java @@ -0,0 +1,372 @@ +package ch.icclab.cyclops.consume.data; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.executor.TaskExecutor; +import ch.icclab.cyclops.load.Loader; +import ch.icclab.cyclops.load.model.PublisherCredentials; +import ch.icclab.cyclops.publish.Messenger; +import ch.icclab.cyclops.timeseries.*; +import ch.icclab.cyclops.util.RegexParser; +import ch.icclab.cyclops.util.loggers.DataLogger; +import com.github.wnameless.json.flattener.JsonFlattener; +import com.google.gson.Gson; +import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; +import org.influxdb.dto.Point; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Process data frame in runnable manner + */ +public class DataProcess implements Runnable { + + private String content; + + private static Messenger messenger = Messenger.getInstance(); + + private PublisherCredentials publisherSettings; + private String defaultMeasurementName; + + private List broadcast; + private BatchPointsContainer container; + private Map> dispatch; + + public static TimeUnit TIME_UNIT = TimeUnit.SECONDS; + + public DataProcess(String content, PublisherCredentials publisher, String measurement) { + this.publisherSettings = publisher; + this.defaultMeasurementName = measurement; + this.container = new BatchPointsContainer(); + this.broadcast = new ArrayList<>(); + this.dispatch = new HashMap<>(); + this.content = content; + } + + @Override + public void run() { + try { + // try to map it as array + List array = new Gson().fromJson(content, List.class); + + // process individual items (cannot be in parallel as we would quickly start thread-context switching) + array.stream().forEach(this::processDataFrame); + + } catch (Exception ignored) { + // this means it was not an array to begin with, just a simple object + processDataFrame(content); + } + + finalise(); + } + + private void finalise() { + + // schedule tasks + TaskExecutor executor = TaskExecutor.getInstance(); + + // persist points + if (container.size() == 1) { + // when only one point in container use shared InfluxDB session for delaying writes + executor.addTask(() -> SharedInfluxDBSession.getSession().persistSinglePoint(container.getFirstPoint())); + } else if (container.size() > 1) { + // we did batch persisting manually so we can store it immediately + executor.addTask(() -> {new InfluxDBClient().persistContainer(container);}); + } + + // broadcast + if (broadcast != null && !broadcast.isEmpty()) { + if (broadcast.size() == 1) { + // broadcast it as value + executor.addTask(() -> {messenger.broadcast(broadcast.get(0));}); + } else { + // broadcast it as array + executor.addTask(() -> {messenger.broadcast(broadcast);}); + } + } + + // dispatch + if (dispatch != null && !dispatch.isEmpty()) { + for (Map.Entry entry: dispatch.entrySet()) { + List list = (List) entry.getValue(); + + if (list.size() == 1) { + // publish it as value + executor.addTask(() -> {messenger.publish(list.get(0), (String) entry.getKey());}); + } else { + // publish it as array + executor.addTask(() -> {messenger.publish(list, (String) entry.getKey());}); + } + } + } + } + + /** + * Process data frame as OBJECT + * @param obj to be processed + */ + private void processDataFrame(Map obj) { + storeAndPublish(obj); + } + + /** + * Process data frame as STRING + * @param str to be processed + */ + private void processDataFrame(String str) { + try { + Map obj = new Gson().fromJson(str, Map.class); + + storeAndPublish(obj); + } catch (Exception ignored) { + // if incoming JSON is not valid we are simply skipping it + } + } + + /** + * Process, store and publish incoming object + * @param obj representation + */ + private void storeAndPublish(Map obj) { + try { + // now parse it and create a database Point + ConsumedData data = processDataAndGetPoint(obj); + + // and finally persist it + container.addPoint(data.getPoint()); + + // publish or broadcast if desired + if (data.shouldPublish()) { + + // where to publish + if (data.doNotBroadcastButRoute()) { + String clazz = data.getClazz(); + + // add to internal dispatch structure + if (dispatch.containsKey(clazz)) { + // we need to fetch the list + List list = dispatch.get(clazz); + list.add(obj); + } else { + // we need to create a new one + List list = new ArrayList<>(); + list.add(obj); + + dispatch.put(clazz, list); + } + + } else { + broadcast.add(obj); + } + } + + } catch (Exception notValidJson) { + DataLogger.log(String.format("Received event/measurement is not a valid JSON: %s", notValidJson.getMessage())); + } + } + + /** + * Data holder for consumed data including database point and mapping guidelines + */ + private class ConsumedData { + private String clazz; + private Point.Builder builder; + private Boolean shouldPublish; + private Boolean doNotBroadcastButRoute; + + public ConsumedData(String clazz, Point.Builder builder, Boolean shouldPublish, Boolean doNotBroadcastButRoute) { + this.clazz = clazz; + this.builder = builder; + this.shouldPublish = shouldPublish; + this.doNotBroadcastButRoute = doNotBroadcastButRoute; + } + + public ConsumedData(String clazz, Point.Builder builder, DataMapping guideline) { + this.clazz = clazz; + this.builder = builder; + this.shouldPublish = guideline.shouldPublish(); + this.doNotBroadcastButRoute = guideline.doNotBroadcastButRoute(); + } + + public String getClazz() { + return clazz; + } + public Point.Builder getPoint() { + return builder; + } + public Boolean shouldPublish() { + return shouldPublish; + } + public Boolean doNotBroadcastButRoute() { + return doNotBroadcastButRoute; + } + } + + /** + * Process and create database Point + * @param map to be processed + * @return point + */ + private ConsumedData processDataAndGetPoint(Map map) { + try { + // check whether type field is present + String clazz = (String) map.get(DataMapping.FIELD_FOR_MAPPING); + + // find corresponding classes + Set set = new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) + -> clazz.equals(RegexParser.getFileName(fileName))).scan(DataConsumer.class.getPackage().getName()); + + // move to the first element + Optional first = set.stream().findFirst(); + + if (first.isPresent()) { + // access object based on provided class definition + DataMapping guideline = (DataMapping) Class.forName(first.get().getName()).newInstance(); + + DataLogger.log(String.format("Event/measurement received with \"%s\" guidelines", clazz)); + + // by default we will work with the original map + Map processed = map; + try { + // let's call user specified pre-processing method + Map preProcessing = guideline.preProcess(map); + + // assign new one if it's valid + if (preProcessing != null && !preProcessing.isEmpty()) { + processed = preProcessing; + } + } catch (Exception ignored) { + } + + // we will be storing flattened version of the map + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(processed)); + + // extract tags from flattened json + Map tags = extractTags(flat, guideline.getTagNames()); + + // parse time field if it was provided + String timeField = guideline.getTimeField(); + Long timeStamp = (timeField != null && flat.containsKey(timeField))? TimeStamp.cast(flat.get(timeField)): null; + if (timeStamp != null && timeStamp >= 0) flat.remove(timeField); + + // determine correct time unit + TimeUnit unit = guideline.getTimeUnit(); + if (unit == null) { + unit = TIME_UNIT; + } + // finally construct database point + Point.Builder builder = constructPoint(clazz, timeStamp, unit, flat, tags); + return new ConsumedData(clazz, builder, guideline); + } else { + // guideline was not present or wasn't valid + throw new Exception(); + } + + } catch (Exception withoutGuidelines){ + // mapping failed, guideline was not provided, use default + DataLogger.log("Event/measurement received without valid guidelines"); + + Point.Builder builder; + + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(map)); + + // even though we don't have mapping template, we still want to have proper measurement name if TYPE is present + if (flat.containsKey(DataMapping.FIELD_FOR_MAPPING)){ + + // get that name + String measurement = (String) flat.get(DataMapping.FIELD_FOR_MAPPING); + + // construct point + builder = constructPoint(measurement, flat); + } else { + builder = constructPoint(defaultMeasurementName, flat); + } + + // determine publisher preferences + Boolean shouldPublish = publisherSettings.getPublisherIncludeAlsoUnknown(); + Boolean doNotRouteButBroadcast = publisherSettings.getPublisherByDefaultDispatchInsteadOfBroadcast(); + String clazz = Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement(); + + return new ConsumedData(clazz, builder, shouldPublish, doNotRouteButBroadcast); + } + } + + /** + * Construct database Point + * @param measurementName to be used + * @param fields as map + * @return point builder + */ + private Point.Builder constructPoint(String measurementName, Map fields) { + return Point.measurement(measurementName).time(System.currentTimeMillis(), TimeUnit.MILLISECONDS).fields(RemoveNullValues.fromMap(fields)); + } + + /** + * Construct database Point + * @param measurementName to be used + * @param timeStamp to be marked + * @param unit of time to be used + * @param fields as map + * @param tags as map + * @return point builder + */ + private Point.Builder constructPoint(String measurementName, Long timeStamp, TimeUnit unit, Map fields, Map tags) { + Point.Builder builder = Point.measurement(measurementName); + + // apply time stamp with time unit + if (timeStamp != null && timeStamp >= 0 && unit != null) { + builder.time(timeStamp, unit); + } else { + builder.time(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + } + + // apply specified tags + if (tags != null && !tags.isEmpty()) { + builder.tag(RemoveNullValues.fromMap(tags)); + } + + // and finally add fields + builder.fields(RemoveNullValues.fromMap(fields)); + + return builder; + } + + /** + * Move tags from content into variable based on provided list of tags + * @param content to be extracted + * @param tagList definition + * @return extracted tags or null + */ + private Map extractTags(Map content, List tagList) { + if (tagList != null) { + Map tags = new HashMap<>(); + + // iterate over list of tags and move them from the original content + for (String tag: tagList) { + tags.put(tag, (String) content.get(tag)); + content.remove(tag); + } + + return tags; + } else { + return null; + } + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/GenericCDR.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/CDR.java similarity index 95% rename from core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/GenericCDR.java rename to core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/CDR.java index 68e4752..368b4dc 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/GenericCDR.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/CDR.java @@ -28,7 +28,7 @@ * Created: 13/05/16 * Description: Generic CDR measurement definition */ -public class GenericCDR implements DataMapping { +public class CDR implements DataMapping { @Override public String getTimeField() { return "time"; @@ -53,7 +53,7 @@ public Map preProcess(Map original) { @Override public Boolean shouldPublish() { - return true; + return false; } @Override diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackCeilometerUDRCDR.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackCeilometerUDRCDR.java deleted file mode 100644 index c5f5de1..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackCeilometerUDRCDR.java +++ /dev/null @@ -1,62 +0,0 @@ -package ch.icclab.cyclops.consume.data.model; - -import ch.icclab.cyclops.consume.data.DataMapping; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - *

- * Created by Manu Perez on 14/07/16. - */ - -public class OpenStackCeilometerUDRCDR implements DataMapping { - @Override - public String getTimeField() { - return "time"; - } - - @Override - public TimeUnit getTimeUnit() { - return TimeUnit.SECONDS; - } - - @Override - public List getTagNames() { - List list = new ArrayList<>(); - list.add("account"); - list.add("meter_name"); - return list; - } - - @Override - public Map preProcess(Map original) { - return original; - } - - @Override - public Boolean shouldPublish() { - return true; - } - - @Override - public Boolean doNotBroadcastButRoute() { - return false; - } -} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackEventCDR.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackEventCDR.java deleted file mode 100644 index 574f502..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackEventCDR.java +++ /dev/null @@ -1,36 +0,0 @@ -package ch.icclab.cyclops.consume.data.model; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -import java.util.ArrayList; -import java.util.List; - - -/** - * Author: Skoviera - * Created: 13/05/16 - * Description: Openstack Event CDR measurement definition - */ -public class OpenStackEventCDR extends GenericCDR { - - @Override - public List getTagNames() { - List list = new ArrayList<>(); - list.add("account"); - list.add("instanceId"); - return list; - } -} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java new file mode 100644 index 0000000..c015654 --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java @@ -0,0 +1,28 @@ +package ch.icclab.cyclops.endpoint; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import org.restlet.resource.ServerResource; + +/** + * Author: Skoviera + * Created: 15/09/16 + * Description: Abstract Endpoint class + */ +public abstract class AbstractEndpoint extends ServerResource { + public abstract String getRoute(); +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java index a37e390..bdddaee 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java @@ -18,32 +18,23 @@ package ch.icclab.cyclops.endpoint; import ch.icclab.cyclops.consume.command.CommandConsumer; -import ch.icclab.cyclops.publish.Messenger; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import com.google.gson.Gson; import org.restlet.representation.Representation; import org.restlet.resource.Post; -import org.restlet.resource.ServerResource; import java.io.IOException; -import java.util.List; /** * Author: Skoviera * Created: 08/07/16 * Description: Handle uploading data frames (the same way as with RabbitMQ) */ -public class CommandEndpoint extends ServerResource { +public class CommandEndpoint extends AbstractEndpoint { - public static String ENDPOINT = "/command"; - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - // logger - final static Logger logger = LogManager.getLogger(CommandEndpoint.class.getName()); + @Override + public String getRoute() { + return "/command"; + } /** * Dispatch and process POST request based on provided parameter @@ -52,7 +43,6 @@ public class CommandEndpoint extends ServerResource { */ @Post public String processPost(Representation entity) throws IOException { - counter.increment(ENDPOINT); try { // first access data consumer @@ -61,20 +51,11 @@ public String processPost(Representation entity) throws IOException { // process the message consumer.consume(entity.getText()); - // get execution status + // get execution status and response CommandConsumer.ExecutionStatus status = consumer.getStatus(); + Object response = consumer.getResponse(); - // was successfully executed - if (status.wasExecuted()) { - - // request restful container - List list = Messenger.getInstance().retrieveRestfulContainer(); - - // return it if it's not empty - return (list != null && !list.isEmpty())? PrettyGson.toJson(list) : status.getMessage(); - } else { - return status.getMessage(); - } + return (status.wasExecuted() && response != null)? new Gson().toJson(response) : status.getMessage(); } catch (Exception e) { return String.format("Error: %s", e.getMessage()); diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java index 5d48d09..b2e96a8 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java @@ -18,36 +18,49 @@ package ch.icclab.cyclops.endpoint; import ch.icclab.cyclops.consume.data.DataConsumer; +import ch.icclab.cyclops.consume.data.DataProcess; +import ch.icclab.cyclops.consume.data.model.CDR; +import ch.icclab.cyclops.dto.Measurement; import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.Settings; +import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.load.model.PublisherCredentials; -import ch.icclab.cyclops.util.APICallCounter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import ch.icclab.cyclops.timeseries.InfluxDBClient; +import ch.icclab.cyclops.timeseries.InfluxDBResponse; +import ch.icclab.cyclops.timeseries.QueryBuilder; +import ch.icclab.cyclops.util.loggers.RESTLogger; +import com.google.gson.Gson; +import org.apache.commons.lang.math.NumberUtils; import org.restlet.representation.Representation; +import org.restlet.resource.Get; import org.restlet.resource.Post; -import org.restlet.resource.ServerResource; import java.io.IOException; +import java.util.List; +import java.util.Map; /** * Author: Skoviera * Created: 08/07/16 * Description: Handle uploading data frames (the same way as with RabbitMQ) */ -public class DataEndpoint extends ServerResource { +public class DataEndpoint extends AbstractEndpoint { - public static String ENDPOINT = "/data"; + private static String SELECTED_MEASUREMENT = CDR.class.getSimpleName(); // used for accessing DataConsumer private PublisherCredentials publisher; private String defaultName; - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); + // supported functions + private final static String FUN_COUNT = "count"; - // logger - final static Logger logger = LogManager.getLogger(DataEndpoint.class.getName()); + // supported params + private final static String PARAM_PAGE = "page"; + private final static String PARAM_FROM = "from"; + private final static String PARAM_TO = "to"; + + private final InfluxDBClient dbClient = new InfluxDBClient(); public DataEndpoint() { Settings settings = Loader.getSettings(); @@ -55,6 +68,144 @@ public DataEndpoint() { this.publisher = settings.getPublisherCredentials(); } + @Override + public String getRoute() { + return "/data"; + } + + @Get + public String processGet(){ + + // access query user specified + Map params = getQuery().getValuesMap(); + + // prepare measurement + Measurement measurement = new Measurement(SELECTED_MEASUREMENT, params); + + // has user provided page number? + Integer pageNumber = 0; + if (params.containsKey(PARAM_PAGE)) { + pageNumber = NumberUtils.toInt((String) params.get(PARAM_PAGE), 0); + params.remove(PARAM_PAGE); + } + + // prepare query + QueryBuilder builder = prepareQuery(measurement, params); + // get number of pages + Integer count = getCountForMeasurement(builder); + + // set page limit + Integer pageLimit = Loader.getSettings().getInfluxDBCredentials().getInfluxDBPageSizeLimit(); + measurement.setPageSize(pageLimit); + + // only if it makes sense + if (count > 0) { + // get data from InfluxDB + fillMeasurementWithRecordsFromDatabase(measurement, builder, pageNumber, pageLimit); + measurement.setTotalRecords(count); + } else { + measurement.setDisplayedRecords(0); + measurement.setTotalRecords(0); + } + + // don't forget to log it + RESTLogger.log(String.format("Serving %d records (out of %d) for measurement \"%s\" as page %d", measurement.getDisplayedRecords(), measurement.getTotalRecords(), measurement.getMeasurement(), measurement.getPageNumber())); + + // return measurement + return new Gson().toJson(measurement); + } + + /** + * Prepare query based on measurement and parameters + * @param measurement details + * @param params to be used + * @return QueryBuilder + */ + private QueryBuilder prepareQuery(Measurement measurement, Map params) { + // create query builder + QueryBuilder builder = new QueryBuilder(measurement.getMeasurement()); + + // date range specification FROM + if (params.containsKey(PARAM_FROM)) { + Long from = NumberUtils.toLong(params.get(PARAM_FROM), 0); + + // only if it makes sense + if (from > 0) { + builder.timeFrom(from, DataProcess.TIME_UNIT); + } + + params.remove(PARAM_FROM); + } + + // date range specification TO + if (params.containsKey(PARAM_TO)) { + Long to = NumberUtils.toLong(params.get(PARAM_TO), 0); + + // only if it makes sense + if (to > 0) { + builder.timeTo(to, DataProcess.TIME_UNIT); + } + + params.remove(PARAM_TO); + } + + // add where clauses iteratively + for (Map.Entry entry: params.entrySet()) { + String strNumber = entry.getValue(); + // if it is number + if (NumberUtils.isNumber(strNumber)) { + builder.where(entry.getKey(), NumberUtils.createDouble(strNumber)); + } else { + builder.where(entry.getKey(), entry.getValue()); + } + } + + return builder; + } + + /** + * Count number of records for specified query + * @param builder prepare query builder + * @return Integer + */ + private Integer getCountForMeasurement(QueryBuilder builder) { + try { + // add COUNT select and pass builder + InfluxDBResponse response = dbClient.executeQuery(builder.count(InfluxDBCredentials.COUNTER_FIELD_NAME)); + List countResult = response.getListOfObjects(); + + // get count from first record + return ((Double) countResult.get(0).get(FUN_COUNT)).intValue(); + } catch (Exception ignored) { + return 0; + } + } + + /** + * Add result of database query to measurement + * @param measurement to be filled + * @param builder prepared query + * @param pageNumber to be used + * @param pageLimit for pagination + */ + private void fillMeasurementWithRecordsFromDatabase(Measurement measurement, QueryBuilder builder, Integer pageNumber, Integer pageLimit) { + + try { + // reset selected fields first + builder.resetSelectClause(); + + // add limit and offset, plus execute query + InfluxDBResponse response = dbClient.executeQuery(builder.limit(pageLimit).offset(pageNumber*pageLimit)); + List parsed = response.getListOfObjects(); + + // update measurement + measurement.setData(parsed); + measurement.setPageNumber(pageNumber); + measurement.setDisplayedRecords(parsed.size()); + } catch (Exception ignored) {} + + } + /** * Dispatch and process POST request based on provided parameter * @param entity json @@ -62,7 +213,6 @@ public DataEndpoint() { */ @Post public String processPost(Representation entity) throws IOException { - counter.increment(ENDPOINT); try { // first access data consumer @@ -71,11 +221,7 @@ public String processPost(Representation entity) throws IOException { // process the message consumer.consume(entity.getText()); - // get stats - Long valid = consumer.getNumberOfValidRecords(); - - - return (valid > 0) ? String.format("%d processed", valid) : "Invalid JSON"; + return "Added for processing"; } catch (Exception e) { return String.format("Error: %s", e.getMessage()); diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java deleted file mode 100644 index 84e6142..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.endpoint; - -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: List available endpoints - */ -public class ListEndpoint extends ServerResource { - - public static String ENDPOINT = "/list"; - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - /** - * This method will return JSON stats for APICallCounter object - * @return JSON - */ - @Get - public String processCommand() { - return PrettyGson.toJson(counter.getAvailableEndpoints()); - } -} \ No newline at end of file diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java deleted file mode 100644 index 7266745..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java +++ /dev/null @@ -1,194 +0,0 @@ -package ch.icclab.cyclops.endpoint; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.consume.data.DataConsumer; -import ch.icclab.cyclops.dto.Measurement; -import ch.icclab.cyclops.load.Loader; -import ch.icclab.cyclops.load.model.InfluxDBCredentials; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.QueryBuilder; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import ch.icclab.cyclops.util.loggers.RESTLogger; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Endpoint for measurements - */ -public class MeasurementEndpoint extends ServerResource{ - final static Logger logger = LogManager.getLogger(MeasurementEndpoint.class.getName()); - - public static String ENDPOINT = "/measurement"; - public static String ATTRIBUTE = "name"; - - // supported functions - private final static String FUN_COUNT = "count"; - - // supported params - private final static String PARAM_PAGE = "page"; - private final static String PARAM_FROM = "from"; - private final static String PARAM_TO = "to"; - - private final APICallCounter counter = APICallCounter.getInstance(); - private final InfluxDBClient dbClient = InfluxDBClient.getInstance(); - private String attribute; - - /** - * Copy parameters for this particular request - */ - public void doInit() { - attribute = (String) getRequestAttributes().get(ATTRIBUTE); - } - - @Get - public String processGet(){ - counter.increment(ENDPOINT); - - // access query user specified - Map params = getQuery().getValuesMap(); - - // prepare measurement - Measurement measurement = new Measurement(attribute, params); - - // has user provided page number? - Integer pageNumber = 0; - if (params.containsKey(PARAM_PAGE)) { - pageNumber = NumberUtils.toInt((String) params.get(PARAM_PAGE), 0); - params.remove(PARAM_PAGE); - } - - // prepare query - QueryBuilder builder = prepareQuery(measurement, params); - // get number of pages - Integer count = getCountForMeasurement(builder); - - // set page limit - Integer pageLimit = Loader.getSettings().getInfluxDBCredentials().getInfluxDBPageSizeLimit(); - measurement.setPageSize(pageLimit); - - // only if it makes sense - if (count > 0) { - // get data from InfluxDB - fillMeasurementWithRecordsFromDatabase(measurement, builder, pageNumber, pageLimit); - measurement.setTotalRecords(count); - } else { - measurement.setDisplayedRecords(0); - measurement.setTotalRecords(0); - } - - // don't forget to log it - RESTLogger.log(String.format("Serving %d records (out of %d) for measurement \"%s\" as page %d", measurement.getDisplayedRecords(), measurement.getTotalRecords(), measurement.getMeasurement(), measurement.getPageNumber())); - - // return measurement - return PrettyGson.toJson(measurement); - } - - /** - * Prepare query based on measurement and parameters - * @param measurement details - * @param params to be used - * @return QueryBuilder - */ - private QueryBuilder prepareQuery(Measurement measurement, Map params) { - // create query builder - QueryBuilder builder = new QueryBuilder(measurement.getMeasurement()); - - // date range specification FROM - if (params.containsKey(PARAM_FROM)) { - Long from = NumberUtils.toLong(params.get(PARAM_FROM), 0); - - // only if it makes sense - if (from > 0) { - builder.timeFrom(from, DataConsumer.TIME_UNIT); - } - - params.remove(PARAM_FROM); - } - - // date range specification TO - if (params.containsKey(PARAM_TO)) { - Long to = NumberUtils.toLong(params.get(PARAM_TO), 0); - - // only if it makes sense - if (to > 0) { - builder.timeTo(to, DataConsumer.TIME_UNIT); - } - - params.remove(PARAM_TO); - } - - // add where clauses iteratively - for (Map.Entry entry: params.entrySet()) { - String strNumber = entry.getValue(); - // if it is number - if (NumberUtils.isNumber(strNumber)) { - builder.where(entry.getKey(), NumberUtils.createDouble(strNumber)); - } else { - builder.where(entry.getKey(), entry.getValue()); - } - } - - return builder; - } - - /** - * Count number of records for specified query - * @param builder prepare query builder - * @return Integer - */ - private Integer getCountForMeasurement(QueryBuilder builder) { - try { - // add COUNT select and pass builder - List countResult = dbClient.executeQuery(builder.count(InfluxDBCredentials.COUNTER_FIELD_NAME)); - - // get count from first record - return ((Double) countResult.get(0).get(FUN_COUNT)).intValue(); - } catch (Exception ignored) { - return 0; - } - } - - /** - * Add result of database query to measurement - * @param measurement to be filled - * @param builder prepared query - * @param pageNumber to be used - * @param pageLimit for pagination - */ - private void fillMeasurementWithRecordsFromDatabase(Measurement measurement, QueryBuilder builder, Integer pageNumber, Integer pageLimit) { - - // reset selected fields first - builder.resetSelectClause(); - - // add limit and offset, plus execute query - List parsed = dbClient.executeQuery(builder.limit(pageLimit).offset(pageNumber*pageLimit)); - - // update measurement - measurement.setData(parsed); - measurement.setPageNumber(pageNumber); - measurement.setDisplayedRecords(parsed.size()); - } -} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java deleted file mode 100644 index 9ab58c7..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java +++ /dev/null @@ -1,73 +0,0 @@ -package ch.icclab.cyclops.endpoint; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.load.Loader; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.QueryBuilder; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Endpoint for measurements - */ -public class MeasurementsEndpoint extends ServerResource{ - final static Logger logger = LogManager.getLogger(MeasurementsEndpoint.class.getName()); - - public static String ENDPOINT = "/measurements"; - - private final APICallCounter counter = APICallCounter.getInstance(); - private final InfluxDBClient dbClient = InfluxDBClient.getInstance(); - - @Get - public String processGet(){ - counter.increment(ENDPOINT); - - // we will list measurements - QueryBuilder builder = QueryBuilder.getMeasurementsQuery(); - - // get list of measurements - List result = dbClient.executeQuery(builder); - - List list = new ArrayList<>(); - - // iterate over list of maps and add to the list only names - for (Map map: result) { - if (map.containsKey(InfluxDBClient.MEASUREMENT_FIELD_NAME)) { - String name = (String) map.get(InfluxDBClient.MEASUREMENT_FIELD_NAME); - - // we are returning only measurement names that are not the default container - if (!name.equals(Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement())) { - list.add(name); - } - } - } - - // return as JSON - return PrettyGson.toJson(list); - } -} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java index 92e603f..0dc32a4 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java @@ -17,25 +17,22 @@ package ch.icclab.cyclops.endpoint; -import ch.icclab.cyclops.util.APICallCounter; import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; /** * Author: Skoviera * Created: 21/01/16 * Description: Serve application's version over root endpoint */ -public class RootEndpoint extends ServerResource { - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - public static String ENDPOINT = "/"; +public class RootEndpoint extends AbstractEndpoint { @Get public String root(){ - counter.increment(ENDPOINT); - return "RCB CDR micro service - version 1.0.1"; + return "RCB BOX micro service - version 2.1.0"; + } + + @Override + public String getRoute() { + return "/"; } } diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java deleted file mode 100644 index ca0cada..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.endpoint; - -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; -import java.util.HashMap; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: This class handles the internal APICallCounter - */ -public class StatusEndpoint extends ServerResource { - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - public static String ENDPOINT = "/status"; - - /** - * This method will return JSON stats for APICallCounter object - * @return JSON - */ - @Get - public String processCommand() { - // first get running statistics - HashMap stats = counter.getRunningStats(); - - // and return - return PrettyGson.toJson(stats); - } -} \ No newline at end of file diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java new file mode 100644 index 0000000..fb01a40 --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java @@ -0,0 +1,61 @@ +package ch.icclab.cyclops.executor; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Global task executor + */ +public class TaskExecutor { + final static Logger logger = LogManager.getLogger(TaskExecutor.class.getName()); + private static TaskExecutor singleton = new TaskExecutor(); + + private ExecutorService executor; + + private TaskExecutor(){ + executor = obtainSession(); + } + + private ExecutorService obtainSession() { + // create a pool threads (based on available cpu cores) + return Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + } + + public static TaskExecutor getInstance() { return singleton; } + + public void addTask(Runnable task) { + if (executor == null) { + executor = obtainSession(); + } + executor.submit(task); + } + + public void shutDown() { + if (executor != null) { + logger.trace("Shutting down Task Executor"); + executor.shutdownNow(); + executor = null; + } + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/publish/APICaller.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/publish/APICaller.java new file mode 100644 index 0000000..2412449 --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/publish/APICaller.java @@ -0,0 +1,117 @@ +package ch.icclab.cyclops.publish; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.util.BeanList; +import com.google.gson.Gson; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.net.URL; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 29/07/16 + * Description: Call API endpoint + */ +public class APICaller { + + public class Response { + private String object; + private int status; + + public Response(String obj, int stat) { + object = obj; + status = stat; + } + + public int getStatus() { + return status; + } + + public String getAsString() throws Exception { + return object; + } + + public List getAsList() throws Exception { + return new Gson().fromJson(object, List.class); + } + + public Map getAsMap() throws Exception { + return new Gson().fromJson(object, Map.class); + } + + public Object getAsClass(Class clazz) throws Exception { + return new Gson().fromJson(object, clazz); + } + + public List getAsListOfType(Class clazz) throws Exception { + List list = new Gson().fromJson(object, List.class); + return BeanList.populate(list, clazz); + } + } + + /** + * Perform POST query and return Response + * @param endpoint to be called + * @param object to be passed + * @return Response object + * @throws Exception + */ + public APICaller.Response post(URL endpoint, Object object) throws Exception { + // prepare connection + HttpClient client = HttpClientBuilder.create().build(); + + // create request + HttpPost request = new HttpPost(endpoint.toURI()); + StringEntity entity = new StringEntity(new Gson().toJson(object)); + request.addHeader("Accept", "application/json"); + request.addHeader("Content-Type", "application/json"); + request.setEntity(entity); + + // execute response + HttpResponse response = client.execute(request); + return new Response(IOUtils.toString(response.getEntity().getContent()), response.getStatusLine().getStatusCode()); + } + + /** + * Perform GET query and return Response + * @param endpoint to be called + * @return Response object + * @throws Exception + */ + public APICaller.Response get(URL endpoint) throws Exception { + // prepare connection + HttpClient client = HttpClientBuilder.create().build(); + + // create request + HttpGet request = new HttpGet(endpoint.toURI()); + request.addHeader("Accept", "application/json"); + request.addHeader("Content-Type", "application/json"); + + // execute response + HttpResponse response = client.execute(request); + return new Response(IOUtils.toString(response.getEntity().getContent()), response.getStatusLine().getStatusCode()); + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/publish/Messenger.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/publish/Messenger.java index 5d7e9dd..796d5e7 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/publish/Messenger.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/publish/Messenger.java @@ -19,6 +19,7 @@ import ch.icclab.cyclops.util.loggers.DispatchLogger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.List; diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java index a582d4e..b1850ee 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java @@ -29,6 +29,7 @@ * Description: Batch session for InfluxDB */ public class BatchPointsContainer { + // if you ever run addPoint in parallel stream, make sure points are encapsulated with Collections.synchronizedList() private List points = new ArrayList<>(); public void addPoint(Point.Builder builder) { @@ -37,7 +38,7 @@ public void addPoint(Point.Builder builder) { public BatchPoints getPoints() { // get empty container - BatchPoints container = InfluxDBClient.getInstance().getEmptyContainer(); + BatchPoints container = new InfluxDBClient().getEmptyContainer(); // iterate over points for (Point.Builder builder : points) { @@ -47,4 +48,12 @@ public BatchPoints getPoints() { return container; } + + public Integer size() { + return points.size(); + } + + public Point.Builder getFirstPoint() { + return (size() > 0) ? points.get(0): null; + } } diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java new file mode 100644 index 0000000..fb87c61 --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java @@ -0,0 +1,118 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import com.github.wnameless.json.flattener.JsonFlattener; +import com.google.gson.Gson; +import org.influxdb.dto.Point; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Author: Skoviera + * Created: 02/09/16 + * Description: Helper class for DB point generation + */ +public class GenerateDBPoint { + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @return Point.Builder + */ + public static Point.Builder fromObject(Object object) { + return generate(object, null, null, null); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param timeField name + * @param unit of time + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTime(Object object, String timeField, TimeUnit unit) { + return generate(object, timeField, unit, null); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param tags list + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTags(Object object, List tags) { + return generate(object, null, null, tags); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param timeField name + * @param unit of time + * @param tags list + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTimeAndTags(Object object, String timeField, TimeUnit unit, List tags) { + return generate(object, timeField, unit, tags); + } + + /** + * Generate a DBPoint + * @param object to be serialised + * @param time field + * @param unit of time + * @param tags list of tags + * @return Point.Builder or null + */ + private static Point.Builder generate(Object object, String time, TimeUnit unit, List tags) { + try { + // flatten possibly nested object + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(object)); + + // create point builder + Point.Builder builder = Point.measurement(object.getClass().getSimpleName()); + + // if time field was provided + if (time != null && unit != null && flat.containsKey(time)) { + builder.time(TimeStamp.cast(flat.get(time)), unit); + flat.remove(time); + } else { + builder.time(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + } + + // if list of tags was provided + if (tags != null && !tags.isEmpty()) { + for (String tag: tags) { + if (flat.containsKey(tag) && flat.get(tag) instanceof String) { + builder.tag(tag, (String) flat.get(tag)); + flat.remove(tag); + } + } + } + + // finally add individual fields + flat.entrySet().stream().filter(entry -> entry.getValue() != null).forEach(entry -> builder.field(entry.getKey(), entry.getValue())); + + return builder; + } catch (Exception ignored) { + return null; + } + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java index 688a6dd..f509425 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java @@ -17,10 +17,13 @@ package ch.icclab.cyclops.timeseries; +import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.util.loggers.TimeSeriesLogger; +import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.atteo.evo.inflector.English; import org.influxdb.InfluxDB; import org.influxdb.InfluxDBFactory; import org.influxdb.dto.BatchPoints; @@ -28,8 +31,10 @@ import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; +import java.util.Collections; import java.util.List; -import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Author: Skoviera @@ -38,38 +43,23 @@ */ public class InfluxDBClient { final static Logger logger = LogManager.getLogger(InfluxDBClient.class.getName()); - public static String MEASUREMENT_FIELD_NAME = "name"; // singleton - private static InfluxDBClient singleton; private InfluxDBCredentials credentials; - - /** - * Create InfluxDB instance - * @param conf to be used - */ - public static void createInstance(InfluxDBCredentials conf){ - if (singleton == null) { - singleton = new InfluxDBClient(conf); - } - } + private InfluxDB session; /** * Constructor * @param conf to be used */ - private InfluxDBClient(InfluxDBCredentials conf) { + public InfluxDBClient(InfluxDBCredentials conf) { credentials = conf; - - createDatabases(credentials.getInfluxDBTSDB()); + session = obtainSession(); } - /** - * Simple implementation of Singleton class - * @return instance of InfluxDB object - */ - public static InfluxDBClient getInstance() { - return singleton; + public InfluxDBClient() { + credentials = Loader.getSettings().getInfluxDBCredentials(); + session = obtainSession(); } /** @@ -80,6 +70,24 @@ private InfluxDB obtainSession() { return InfluxDBFactory.connect(credentials.getInfluxDBURL(), credentials.getInfluxDBUsername(), credentials.getInfluxDBPassword()); } + /** + * Ping InfluxDB server to see whether it is alive + * @throws Exception + */ + public void ping() throws Exception { + session.ping(); + } + + /** + * Enable batch processing for items that are added as single points + * @param flushPoints flush every X points (for example 2000) + * @param flushFrequency flush every Y time unit (for example 100 ms) + * @param unit time unit (for example milliseconds) + */ + public void configureBatchOnSinglePoints(Integer flushPoints, Integer flushFrequency, TimeUnit unit) { + session.enableBatch(flushPoints, flushFrequency, unit); + } + /** * Save container to InfluxDB * @param container to be persisted @@ -88,9 +96,8 @@ public void persistContainer(BatchPointsContainer container) { persistContainer(container.getPoints()); } private void persistContainer(BatchPoints container) { - InfluxDB con = obtainSession(); TimeSeriesLogger.log(String.format("Saving container with %d points to database", container.getPoints().size())); - con.write(container); + session.write(container); } /** @@ -106,14 +113,11 @@ protected BatchPoints getEmptyContainer() { * @param builder to generate point */ public void persistSinglePoint(Point.Builder builder) { - BatchPoints container = getEmptyContainer(); - // add mandatory hidden field Point point = builder.addField(InfluxDBCredentials.COUNTER_FIELD_NAME, true).build(); - container.point(point); - - persistContainer(container); + // depending on whether batch processing for single points is enabled store immediately or wait for flush + session.write(credentials.getInfluxDBTSDB(), "default", point); } /** @@ -121,41 +125,38 @@ public void persistSinglePoint(Point.Builder builder) { * @param names for database creation */ public void createDatabases(String ... names) { - InfluxDB client = obtainSession(); - // now create required databases for (String name: names) { TimeSeriesLogger.log(String.format("Making sure \"%s\" database exists", name)); - client.createDatabase(name); + session.createDatabase(name); } } /** * Execute query * @param builder QueryBuilder - * @return QueryResult + * @return InfluxDBResponse or null */ - public List executeQuery(QueryBuilder builder) { - InfluxDB client = obtainSession(); + public InfluxDBResponse executeQuery(QueryBuilder builder) { + return executeQuery(Collections.singletonList(builder)); + } + public InfluxDBResponse executeQuery(List builders) { + try { + TimeSeriesLogger.log(String.format("About to execute %d %s", builders.size(), English.plural("query", builders.size()))); - // execute query - QueryResult result = client.query(new Query(builder.build(), credentials.getInfluxDBTSDB())); + // concatenate multiple queries into one + List queries = builders.stream().map(QueryBuilder::build).collect(Collectors.toList()); + String multipleQuery = StringUtils.join(queries, ";"); - // return it parsed as list of maps - return ParseQueryResult.parse(result); - } + // connect to InfluxDB and execute query + QueryResult result = session.query(new Query(multipleQuery, credentials.getInfluxDBTSDB())); - /** - * Execute query and map it to specified class - * @param builder query - * @param clazz for mapping - * @return list or null - */ - public List executeQueryAndMapItToClass(QueryBuilder builder, Class clazz) { - // first execute query and get results - List result = executeQuery(builder); + // return InfluxDB response or null + return (!result.hasError())? new InfluxDBResponse(result): null; - // now return it parsed - return ParseQueryResult.map(result, clazz); + } catch (Exception ignored) { + TimeSeriesLogger.log(String.format("Query execution failed: %s", ignored.getMessage())); + return null; + } } } diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java new file mode 100644 index 0000000..f94abac --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java @@ -0,0 +1,136 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.load.model.InfluxDBCredentials; +import ch.icclab.cyclops.util.BeanList; +import com.github.wnameless.json.unflattener.JsonUnflattener; +import com.google.gson.Gson; +import org.influxdb.dto.QueryResult; +import org.joda.time.DateTime; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: InfluxDB response container + */ +public class InfluxDBResponse { + + private static String TIME_FIELD = "time"; + + private QueryResult object; + + public InfluxDBResponse(QueryResult object) { + this.object = object; + } + + /** + * Output underlying InfluxDB structure in JSON + * @return flat table structure + */ + public String getTableAsJson() throws Exception { + return new Gson().toJson(this.object); + + } + + /** + * Parse and transform InfluxDB's underlying structure + * @return JSON + */ + public String getListOfObjectsAsJson() throws Exception { + return new Gson().toJson(getListOfObjects()); + } + + /** + * Parse and transform InfluxDB's underlying structure + * @param clazz definition + * @return List + */ + public List getAsListOfType(Class clazz) throws Exception { + List list = getListOfObjects(); + return BeanList.populate(list, clazz); + } + + /** + * Parse and transform InfluxDB's underlying structure + * @return list of maps + */ + public List getListOfObjects() throws Exception { + + List list = new ArrayList<>(); + + if (object != null && !object.hasError()) { + // iterate over available results + for (QueryResult.Result result: object.getResults()) { + // multiple query responses can have empty results + if (result != null) { + List bulk = result.getSeries(); + if (bulk != null && !bulk.isEmpty()) { + // iterate over available series + for (QueryResult.Series series: result.getSeries()) { + // iterate over individual value sets + Map tags = series.getTags(); + List columns = series.getColumns(); + int time = columns.indexOf(TIME_FIELD); + List> values = series.getValues(); + + // iterate over individual values + if (values != null && !values.isEmpty()) { + for (List value: values) { + // we will store one row here + Map row = new HashMap<>(); + + // add tag values if there are any + if (tags != null && !tags.isEmpty()) { + row.putAll(tags); + } + + // individual entries of the value + int i = 0; + for (Object entry: value) { + // timestamp needs to be in Long and not String + if (time == i) { + entry = new DateTime(entry).getMillis() / 1000; + } + + row.put(columns.get(i), entry); + i++; + } + + // remove property that is used for counting and should never be visible + row.remove(InfluxDBCredentials.COUNTER_FIELD_NAME); + + // return de-flattened map + list.add(new Gson().fromJson(JsonUnflattener.unflatten(new Gson().toJson(row)), Map.class)); + } + } + } + } + } + } + + return list; + } + + throw new Exception("Couldn't parse InfluxDB response"); + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java deleted file mode 100644 index 4ba0769..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java +++ /dev/null @@ -1,128 +0,0 @@ -package ch.icclab.cyclops.timeseries; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.load.model.InfluxDBCredentials; -import com.github.wnameless.json.unflattener.JsonUnflattener; -import com.google.gson.Gson; -import org.apache.commons.beanutils.BeanUtils; -import org.influxdb.dto.QueryResult; -import org.joda.time.DateTime; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Parse InfluxDB's Query Result - */ -public class ParseQueryResult { - - private static String TIME_FIELD = "time"; - - public static List parse(QueryResult data) { - // container for result - List container = new ArrayList<>(); - - // mapping - Gson gson = new Gson(); - - try { - // go over all results - for (QueryResult.Result result : data.getResults()) { - - // go over individual series - for (QueryResult.Series serie : result.getSeries()) { - - // store columns here - List columns = new ArrayList<>(); - columns.addAll(serie.getColumns()); - - // get time field position - int time = columns.indexOf(TIME_FIELD); - - // iterate over values - for (List values : serie.getValues()) { - // we will store one row here - Map row = new HashMap<>(); - - // add tag values if there are any - Map tags = serie.getTags(); - if (tags != null && !tags.isEmpty()) { - row.putAll(tags); - } - - // access objects - int i = 0; - for (Object value: values) { - - // timestamp needs to be in Long and not String - if (time == i) { - try { - // return seconds not milliseconds - value = new DateTime(value).getMillis() / 1000; - } catch (Exception ignored) { - } - } - - row.put(columns.get(i), value); - i++; - } - - // delete meta field - row.remove(InfluxDBCredentials.COUNTER_FIELD_NAME); - - // de-flatten structure - String rich = JsonUnflattener.unflatten(gson.toJson(row)); - container.add(gson.fromJson(rich, Map.class)); - } - } - } - } catch (Exception ignored) { - // in case of empty QueryResult body do nothing - } - - return container; - } - - public static List map(List list, Class clazz) { - List mapped = new ArrayList<>(); - - // iterate and map those objects - if (list != null) { - for (Map map : list) { - try { - Object bean = clazz.newInstance(); - - // map HashMap to POJO - BeanUtils.populate(bean, map); - - // add it to list of mapped CDRs - mapped.add(bean); - - } catch (Exception ignored) { - return null; - } - } - } - - return mapped; - } -} diff --git a/core/billing/src/main/java/ch/icclab/cyclops/util/PrettyGson.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java similarity index 61% rename from core/billing/src/main/java/ch/icclab/cyclops/util/PrettyGson.java rename to core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java index d54d16a..f64855f 100644 --- a/core/billing/src/main/java/ch/icclab/cyclops/util/PrettyGson.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java @@ -1,4 +1,4 @@ -package ch.icclab.cyclops.util; +package ch.icclab.cyclops.timeseries; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -16,21 +16,22 @@ * under the License. */ -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import java.util.concurrent.TimeUnit; /** * Author: Skoviera - * Created: 08/03/16 - * Description: Always use pretty JSON output + * Created: 31/08/16 + * Description: Shared InfluxDB session */ -public class PrettyGson { +public class SharedInfluxDBSession { + private static InfluxDBClient session; - private static Gson getGson() { - return new GsonBuilder().setPrettyPrinting().create(); - } + public static InfluxDBClient getSession() { + if (session == null) { + session = new InfluxDBClient(); + session.configureBatchOnSinglePoints(2000, 100, TimeUnit.MILLISECONDS); + } - public static String toJson(Object object) { - return getGson().toJson(object); + return session; } } diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java new file mode 100644 index 0000000..f77a4db --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java @@ -0,0 +1,53 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import java.math.BigDecimal; + +/** + * Author: Skoviera + * Created: 06/09/16 + * Description: Various utils for TimeStamp object manipulation + */ +public class TimeStamp { + + /** + * Parse long from unknown type + * @param unknown object + * @return Long or null + */ + public static Long cast(Object unknown) { + try { + BigDecimal dec = (BigDecimal) unknown; + return dec.longValue(); + } catch (Exception a) { + try { + return Long.parseLong((String) unknown); + } catch (Exception b) { + try { + return (Long) unknown; + } catch (Exception c) { + try { + return Long.valueOf(unknown.toString()); + } catch (Exception d) { + return null; + } + } + } + } + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/APICallCounter.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/APICallCounter.java deleted file mode 100644 index ff640c1..0000000 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/APICallCounter.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.util; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: Class that will count API Calls and then provide the information (and reset counters) - */ -public class APICallCounter { - // we need a singleton for this - private static APICallCounter singleton = new APICallCounter(); - - // Hash-map to be used for counting - private HashMap map; - - // list used for holding them all - private ArrayList list; - - // Hash-map used as a template (for registering fields) - private HashMap template; - - /** - * Private Constructor is necessary for the singleton model - */ - private APICallCounter() { - template = new HashMap<>(); - list = new ArrayList<>(); - map = new HashMap<>(); - } - - /** - * Simply return singleton instance - * @return APICallCounter instance - */ - public static APICallCounter getInstance() { - return singleton; - } - - /** - * Will increment the counter for provided key - * In case that this key was not previously registered, it will add it to the hash-map - * However on dump, it will get removed and only the registered ones will stay - * @param key meaning API endpoint - */ - public void increment(String key) { - Object value = map.get(key); - - // now save new value or increment it - if (value == null) { - map.put(key, 1); - } else { - map.put(key, ((Integer) value) + 1); - } - } - - /** - * Register an API endpoint to the counter - * @param key as endpoint string - */ - public void registerEndpoint(String key) { - template.put(key, 0); - map.put(key, 0); - list.add(key); - } - - /** - * Register an API endpoint for listing - * @param key as endpoint - */ - public void registerEndpointWithoutCounting(String key) { - list.add(key); - } - - /** - * Will ask for running stats and DUMP the old hash-map so it can start from zero again - * @return hash-map - */ - public HashMap getRunningStats() { - // make a deep copy - HashMap stats = new HashMap(); - stats.putAll(map); - - // reset map based on registered entries in template - map = new HashMap(); - map.putAll(template); - - // return deep copy - return stats; - } - - /** - * Return list of available endpoints - * @return list - */ - public List getAvailableEndpoints() { - return list; - } -} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/BeanList.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/BeanList.java new file mode 100644 index 0000000..6c9d265 --- /dev/null +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/BeanList.java @@ -0,0 +1,54 @@ +package ch.icclab.cyclops.util; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.beanutils.BeanUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Utility to populate list of beans + */ +public class BeanList { + public static List populate(List list, Class clazz) { + List mapped = new ArrayList<>(); + + // iterate and map those objects + if (list != null) { + for (Map map : list) { + try { + Object bean = clazz.newInstance(); + + // map HashMap to POJO + BeanUtils.populate(bean, map); + + // add it to list of mapped CDRs + mapped.add(bean); + + } catch (Exception ignored) { + return null; + } + } + } + + return mapped; + } +} diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java b/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java index b21669b..7c61a5e 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java +++ b/core/rc/cdr/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java @@ -17,6 +17,7 @@ */ import ch.icclab.cyclops.consume.RabbitMQListener; +import ch.icclab.cyclops.executor.TaskExecutor; import ch.icclab.cyclops.publish.RabbitMQPublisher; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,7 +33,10 @@ public class ShutDownListener extends Thread{ @Override public void run() { - logger.trace("We are shutting down CDR micro service"); + logger.trace("We are shutting down BOX micro service"); + + // shut down executor service + TaskExecutor.getInstance().shutDown(); // and Consumer (RabbitMQ) RabbitMQListener.shutDown(); diff --git a/core/rc/cdr/src/main/resources/log4j2.component.properties b/core/rc/cdr/src/main/resources/log4j2.component.properties new file mode 100644 index 0000000..683a3da --- /dev/null +++ b/core/rc/cdr/src/main/resources/log4j2.component.properties @@ -0,0 +1 @@ +Log4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector \ No newline at end of file diff --git a/core/rc/cdr/src/main/resources/log4j2.xml b/core/rc/cdr/src/main/resources/log4j2.xml index aca3037..d5d7c36 100644 --- a/core/rc/cdr/src/main/resources/log4j2.xml +++ b/core/rc/cdr/src/main/resources/log4j2.xml @@ -1,32 +1,32 @@ - - - - - - - + + + + + + + - - + + - - + + - - + + - - + + - + - + - + diff --git a/core/rc/rate/ReleaseNotes.txt b/core/rc/rate/ReleaseNotes.txt new file mode 100644 index 0000000..bb483be --- /dev/null +++ b/core/rc/rate/ReleaseNotes.txt @@ -0,0 +1,7 @@ +Version: 1.1.0 +Date: 16/September/2016 +Notes: Support for UDR envelope (including Usage Data), outputting rated Charge Data (in CDR envelope) + +Version: 1.0.0 +Date: 1/July/2016 +Notes: Initial release of the RCB Cyclops Rating micro service \ No newline at end of file diff --git a/core/rc/rate/bin/rate.jar b/core/rc/rate/bin/rate.jar index 5b134fe..cdf00d0 100644 Binary files a/core/rc/rate/bin/rate.jar and b/core/rc/rate/bin/rate.jar differ diff --git a/core/rc/rate/config/rate.conf b/core/rc/rate/config/rate.conf index 557724c..7bfbafb 100644 --- a/core/rc/rate/config/rate.conf +++ b/core/rc/rate/config/rate.conf @@ -21,7 +21,6 @@ ConsumerDataQueue=cyclops.rate.consume UsageField=usage ChargeField=charge ChargeSuffix=CDR -RateField=rate DefaultRate=0.5 # Rates for specified sources diff --git a/core/rc/rate/pom.xml b/core/rc/rate/pom.xml index ef81cce..6aae78a 100644 --- a/core/rc/rate/pom.xml +++ b/core/rc/rate/pom.xml @@ -7,7 +7,7 @@ ch.icclab.cyclops.rate cyclops-rate jar - 0.0.1 + 1.1.0 Rate diff --git a/core/rc/rate/scripts/compile.sh b/core/rc/rate/scripts/compile.sh index b6d87a0..a432da6 100755 --- a/core/rc/rate/scripts/compile.sh +++ b/core/rc/rate/scripts/compile.sh @@ -26,4 +26,4 @@ mvn dependency:tree mvn package assembly:single cd target -mv cyclops-rate-0.0.1-jar-with-dependencies.jar ../bin/rate.jar +mv cyclops-rate-1.1.0-jar-with-dependencies.jar ../bin/rate.jar diff --git a/core/rc/rate/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java b/core/rc/rate/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java index 6342e68..5b7a222 100644 --- a/core/rc/rate/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java +++ b/core/rc/rate/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger; import java.util.*; -import java.util.concurrent.TimeUnit; /** * Author: Skoviera @@ -40,10 +39,12 @@ protected void consume(String content) { // make sure there is something to be rated at all if (array != null && !array.isEmpty()) { // now apply rates - List rated = rateAndCharge(array); + List rated = process(array); // push it to the next step - publishOrBroadcast(rated); + if (rated != null && !rated.isEmpty()) { + publishOrBroadcast(rated); + } } } catch (Exception ignored) { @@ -53,9 +54,11 @@ protected void consume(String content) { if (obj != null) { // apply rate - Map rated = rateAndCharge(obj); + Map rated = process(obj); - publishOrBroadcast(Collections.singletonList(rated)); + if (rated != null && !rated.isEmpty()) { + publishOrBroadcast(Collections.singletonList(rated)); + } } } catch (Exception ignoredAgain) {} } @@ -66,47 +69,117 @@ protected void consume(String content) { * @param obj as Map * @return object or null */ - private Map rateAndCharge(Map obj) { - // make sure we have usage field present - if (obj.containsKey(rating.getUsageField())) { + private Map process(Map obj) { - // normalise usage object - Double usage = getUsage(obj.get(rating.getUsageField())); + if (obj.containsKey(RatingPreferences.CLASS_FIELD_NAME) && obj.get(RatingPreferences.CLASS_FIELD_NAME).equals(RatingPreferences.NEW_UDR_CLASS_NAME)) { + // rate new UDR format + return rateNewUDREnvelopeFormat(obj); - // find correct rate or use default one - Double rate = getRate(obj); + } else if (obj.containsKey(rating.getUsageField())) { + // rate record + rateIndividualItem(obj, true); + // return updated hashmap + return obj; - // calculate charge - Double charge = usage * rate; + } else { + return null; + } + } + + private Map rateNewUDREnvelopeFormat(Map obj) { + try { + HashMap map = new HashMap<>(obj); + HashMap result = new HashMap<>(); - // put rate and charge back - obj.put(rating.getRateField(), rate); - obj.put(rating.getChargeField(), charge); + Double charge = 0d; + Boolean found = false; + + // find field that is of a List type + Iterator> iterator = map.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + + if (entry.getValue() instanceof List) { + List list = (List) entry.getValue(); + List container = new ArrayList<>(); + + Boolean update = false; + + for (Object item: list) { + // now we know we can rate it + if (item instanceof Map && ((Map) item).containsKey(RatingPreferences.CLASS_FIELD_NAME) + && ((Map) item).containsKey(RatingPreferences.DEFAULT_USAGE_FIELD)) { + + // rate and return calculated charge + charge += rateIndividualItem((Map) item, false); + + update = true; + found = true; + } + + // add it to container + container.add(item); + } - // add string prefix to charge record - if (obj.containsKey(RatingPreferences.CLASS_FIELD_NAME)) { - obj.put(RatingPreferences.CLASS_FIELD_NAME, String.format("%s%s", obj.get(RatingPreferences.CLASS_FIELD_NAME), rating.getChargeSuffix())); + if (update) { + // set entry to contain list of updated Charge Records + result.put(entry.getKey(), container); + // also update overall charge of the whole CDR + result.put(rating.getChargeField(), charge); + } + } + else { + // we still need to copy values + result.put(entry.getKey(), entry.getValue()); + } } - return obj; - } else { + // update _class name to CDR + result.put(RatingPreferences.CLASS_FIELD_NAME, RatingPreferences.DEFAULT_CHARGE_SUFFIX); + + // make sure we are returning updated map only if something has changed + return (found)? result: null; + } catch (Exception e) { return null; } + + } + + private Double rateIndividualItem(Map obj, Boolean suffix) { + // normalise usage object + Double usage = getUsage(obj.get(rating.getUsageField())); + + // find correct rate or use default one + Double rate = getRate(obj); + + // calculate charge + Double charge = usage * rate; + + // put rate and charge back + obj.put(rating.getChargeField(), charge); + + // add string suffix to charge record + if (suffix && obj.containsKey(RatingPreferences.CLASS_FIELD_NAME)) { + obj.put(RatingPreferences.CLASS_FIELD_NAME, String.format("%s%s", obj.get(RatingPreferences.CLASS_FIELD_NAME), rating.getChargeSuffix())); + } + + return charge; } + /** * Rate and Charge list of UDR records * @param list to be rated * @return rated list */ - private List rateAndCharge(List list) { + private List process(List list) { List ratedList = new ArrayList<>(); // iterate and rate all objects for (Map obj: list) { // rate the object - Map rated = rateAndCharge(obj); + Map rated = process(obj); // only add to the list if it was successfully rated if (rated != null) { diff --git a/core/rc/rate/src/main/java/ch/icclab/cyclops/load/Settings.java b/core/rc/rate/src/main/java/ch/icclab/cyclops/load/Settings.java index de73d4e..fb1b982 100644 --- a/core/rc/rate/src/main/java/ch/icclab/cyclops/load/Settings.java +++ b/core/rc/rate/src/main/java/ch/icclab/cyclops/load/Settings.java @@ -175,12 +175,6 @@ private RatingPreferences loadRatingPreferences() { } ratingPreferences.setChargeSuffix(chargeSuffix); - String rate = properties.getProperty("RateField"); - if (rate == null || rate.isEmpty()) { - rate = RatingPreferences.DEFAULT_RATE_FIELD; - } - ratingPreferences.setRateField(rate); - ratingPreferences.setDefaultRate(NumberUtils.toDouble(properties.getProperty("DefaultRate"), RatingPreferences.DEFAULT_RATE_VALUE)); return ratingPreferences; diff --git a/core/rc/rate/src/main/java/ch/icclab/cyclops/load/model/RatingPreferences.java b/core/rc/rate/src/main/java/ch/icclab/cyclops/load/model/RatingPreferences.java index e557108..d912c80 100644 --- a/core/rc/rate/src/main/java/ch/icclab/cyclops/load/model/RatingPreferences.java +++ b/core/rc/rate/src/main/java/ch/icclab/cyclops/load/model/RatingPreferences.java @@ -25,8 +25,9 @@ public class RatingPreferences { public static String DEFAULT_USAGE_FIELD = "usage"; public static String DEFAULT_CHARGE_FIELD = "charge"; - public static String DEFAULT_RATE_FIELD = "rate"; + public static String DEFAULT_CHARGE_SUFFIX = "CDR"; + public static String NEW_UDR_CLASS_NAME = "UDR"; public static Double DEFAULT_RATE_VALUE = 1.0d; public static String CLASS_FIELD_NAME = "_class"; @@ -34,7 +35,6 @@ public class RatingPreferences { // These fields correspond with the configuration file private String usageField; private String chargeField; - private String rateField; private Double defaultRate; private String chargeSuffix; @@ -56,14 +56,6 @@ public void setChargeField(String chargeField) { this.chargeField = chargeField; } - public String getRateField() { - return rateField; - } - - public void setRateField(String rateField) { - this.rateField = rateField; - } - public Double getDefaultRate() { return defaultRate; } diff --git a/core/rc/rate/src/main/resources/log4j2.xml b/core/rc/rate/src/main/resources/log4j2.xml index 4066200..21e4828 100644 --- a/core/rc/rate/src/main/resources/log4j2.xml +++ b/core/rc/rate/src/main/resources/log4j2.xml @@ -1,16 +1,16 @@ - + - + - + - + diff --git a/core/udr/ReleaseNotes.txt b/core/udr/ReleaseNotes.txt new file mode 100644 index 0000000..edd8121 --- /dev/null +++ b/core/udr/ReleaseNotes.txt @@ -0,0 +1,7 @@ +Version: 2.1.0 +Date: 16/September/2016 +Notes: Data ingestion and query execution optimisations, new UDR output format (part of GenerateUDR command) + +Version: 2.0.0 +Date: 1/July/2016 +Notes: Initial release of the RCB Cyclops UDR micro service \ No newline at end of file diff --git a/core/udr/bin/udr.jar b/core/udr/bin/udr.jar index 6ed9c03..884c444 100644 Binary files a/core/udr/bin/udr.jar and b/core/udr/bin/udr.jar differ diff --git a/core/udr/config/udr.conf b/core/udr/config/udr.conf index e099c8a..f364861 100644 --- a/core/udr/config/udr.conf +++ b/core/udr/config/udr.conf @@ -28,4 +28,4 @@ ConsumerPassword=guest ConsumerPort=5672 ConsumerVirtualHost=/ ConsumerDataQueue=cyclops.udr.consume -ConsumerCommandsQueue=cyclops.udr.commands \ No newline at end of file +ConsumerCommandsQueue=cyclops.udr.commands diff --git a/core/udr/pom.xml b/core/udr/pom.xml index 167ab23..eb4955d 100644 --- a/core/udr/pom.xml +++ b/core/udr/pom.xml @@ -7,19 +7,20 @@ ch.icclab.cyclops.udr cyclops-udr jar - 0.0.1 + 2.1.0 UDR 1.8 UTF-8 UTF-8 - 2.3.4 - 2.9.3 - 2.5 - 3.6.1 - 1.7.12 + 2.3.7 + 2.9.4 + 3.1.0 + 3.6.5 + 1.7.21 2.2 + 2.6.2 @@ -44,6 +45,7 @@ org.apache.maven.plugins maven-assembly-plugin + 2.6 jar-with-dependencies @@ -102,7 +104,7 @@ javax.servlet - servlet-api + javax.servlet-api ${servlet-version} @@ -118,47 +120,47 @@ org.slf4j jul-to-slf4j - 1.7.12 + ${slf4j-version} runtime org.apache.logging.log4j log4j-api - 2.3 + ${log4j-version} org.apache.logging.log4j log4j-core - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-web - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-jul - 2.3 + ${log4j-version} runtime org.apache.logging.log4j log4j-slf4j-impl - 2.3 + ${log4j-version} runtime org.json json - 20140107 + 20160212 com.google.code.gson gson - 2.6.2 + 2.7 io.gsonfire @@ -178,17 +180,32 @@ com.metapossum metapossum-scanner - 1.0 + 1.0.1 com.github.wnameless json-flattener - 0.1.6 + 0.2.2 commons-beanutils commons-beanutils 1.9.2 + + org.apache.httpcomponents + httpclient + 4.5.2 + + + com.lmax + disruptor + 3.3.5 + + + org.atteo + evo-inflector + 1.2.1 + \ No newline at end of file diff --git a/core/udr/scripts/compile.sh b/core/udr/scripts/compile.sh index 2f6fd82..6fca743 100755 --- a/core/udr/scripts/compile.sh +++ b/core/udr/scripts/compile.sh @@ -26,4 +26,4 @@ mvn dependency:tree mvn package assembly:single cd target -mv cyclops-udr-0.0.1-jar-with-dependencies.jar ../bin/udr.jar +mv cyclops-udr-2.1.0-jar-with-dependencies.jar ../bin/udr.jar diff --git a/core/udr/src/main/java/ch/icclab/cyclops/application/Main.java b/core/udr/src/main/java/ch/icclab/cyclops/application/Main.java index 43eb9b4..ac509d6 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/application/Main.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/application/Main.java @@ -21,6 +21,7 @@ import ch.icclab.cyclops.consume.data.DataConsumer; import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.Settings; +import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.load.model.PublisherCredentials; import ch.icclab.cyclops.publish.RabbitMQPublisher; import ch.icclab.cyclops.timeseries.InfluxDBClient; @@ -38,6 +39,11 @@ */ public class Main extends Application{ + static { + // Nothing can appear before this initializer + System.setProperty("Log4jContextSelector", "org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"); + } + final static Logger logger = LogManager.getLogger(Main.class.getName()); private static final int OK_HELP = 1; @@ -250,8 +256,13 @@ private static void checkCustomPortOption(String[] args) { */ private static void checkAndConfigureInfluxDB() { try { - logger.trace("Binding to InfluxDB"); - InfluxDBClient.createInstance(Loader.getSettings().getInfluxDBCredentials()); + logger.trace("Binding to InfluxDB and creating databases"); + InfluxDBCredentials credentials = Loader.getSettings().getInfluxDBCredentials(); + InfluxDBClient client = new InfluxDBClient(credentials); + + client.ping(); + client.createDatabases(credentials.getInfluxDBTSDB()); + } catch (Exception e) { String log = String.format("Couldn't connect to InfluxDb: %s", e.getMessage()); logger.error(log); diff --git a/core/udr/src/main/java/ch/icclab/cyclops/application/Service.java b/core/udr/src/main/java/ch/icclab/cyclops/application/Service.java index cfc1bc4..9af27f0 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/application/Service.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/application/Service.java @@ -18,12 +18,16 @@ package ch.icclab.cyclops.application; import ch.icclab.cyclops.endpoint.*; -import ch.icclab.cyclops.util.APICallCounter; +import ch.icclab.cyclops.util.RegexParser; +import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.restlet.Restlet; import org.restlet.routing.Router; +import java.util.ArrayList; +import java.util.List; + /** * Author: Skoviera * Created: 21/01/16 @@ -36,63 +40,46 @@ public class Service { // Router for registering api endpoints private Router router; - // API counter - private APICallCounter counter; - /** - * Attaching endpoints to router + * This method handles the incoming request and routes it to the appropriate resource class */ - private void attachTheRest() { - logger.trace("Attaching measurement endpoint"); - router.attach(String.format("%s/{%s}", MeasurementEndpoint.ENDPOINT, MeasurementEndpoint.ATTRIBUTE), MeasurementEndpoint.class); - counter.registerEndpoint(MeasurementEndpoint.ENDPOINT); - - router.attach(MeasurementsEndpoint.ENDPOINT, MeasurementsEndpoint.class); - counter.registerEndpoint(MeasurementsEndpoint.ENDPOINT); + public Restlet createInboundRoot() throws Exception { - logger.trace("Attaching data endpoint"); - router.attach(DataEndpoint.ENDPOINT, DataEndpoint.class); - counter.registerEndpoint(DataEndpoint.ENDPOINT); + logger.trace("Initialising UDR microservice and creating routes"); - logger.trace("Attaching command endpoint"); - router.attach(CommandEndpoint.ENDPOINT, CommandEndpoint.class); - counter.registerEndpoint(CommandEndpoint.ENDPOINT); - } + router = attachRoutes(); - /** - * Construct application by accessing context, creating router and counter - */ - private void initialiseApplication() { - logger.trace("Initialising UDR microservice"); + logger.trace("Routes for UDR microservice successfully created"); - router = new Router(); - counter = APICallCounter.getInstance(); + return router; } /** - * This method handles the incoming request and routes it to the appropriate resource class + * Attach routes that extend AbstractEndpoint class + * @return router */ - public Restlet createInboundRoot() throws Exception { - - // let's start by initialising and loading configuration settings - initialiseApplication(); - - logger.trace("Creating routes for UDR microservice"); - - // root, status and list endpoints - router.attach(RootEndpoint.ENDPOINT, RootEndpoint.class); - counter.registerEndpoint(RootEndpoint.ENDPOINT); - - router.attach(StatusEndpoint.ENDPOINT, StatusEndpoint.class); - counter.registerEndpointWithoutCounting(StatusEndpoint.ENDPOINT); - - router.attach(ListEndpoint.ENDPOINT, ListEndpoint.class); - counter.registerEndpointWithoutCounting(ListEndpoint.ENDPOINT); - - // attach other endpoints - attachTheRest(); - - logger.trace("Routes for UDR microservice successfully created"); + private Router attachRoutes() { + Router router = new Router(); + + List list = new ArrayList<>(); + + try { + // find all endpoints and add them to the list + list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) -> !AbstractEndpoint.class.getSimpleName(). + equals(RegexParser.getFileName(fileName))).scan(AbstractEndpoint.class.getPackage().getName())); + } catch (Exception ignored) {} + + // create routes + for (Class clazz : list) { + // only those which extend Endpoint + if (AbstractEndpoint.class.isAssignableFrom(clazz)) { + try { + // get endpoint's route path + String route = ((AbstractEndpoint) clazz.newInstance()).getRoute(); + router.attach(route, clazz); + } catch (Exception ignored) {} + } + } return router; } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/Command.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/Command.java index 5419bb0..6616256 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/Command.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/Command.java @@ -27,8 +27,9 @@ public abstract class Command { /** * Every command has to implement execute method + * @return Object of your selection or Null */ - protected abstract void execute(); + protected abstract Object execute(); //===== Getters and Setters public String get_class() { diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java index f3f5b83..3c01713 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/CommandConsumer.java @@ -10,6 +10,8 @@ */ public class CommandConsumer extends AbstractConsumer { + private Object response; + private ExecutionStatus status; public class ExecutionStatus { private Boolean executed; @@ -42,7 +44,7 @@ public void consume(String content) { if (command != null) { try { - command.execute(); + response = command.execute(); status = new ExecutionStatus(true, String.format("[OK] command \"%s\" successfully executed", command.get_class())); CommandLogger.log(status.getMessage()); @@ -59,4 +61,7 @@ public void consume(String content) { public ExecutionStatus getStatus() { return status; } + public Object getResponse() { + return response; + } } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java index 66ed0e2..fdc8543 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/CommandMapping.java @@ -37,25 +37,25 @@ public class CommandMapping { final static Logger logger = LogManager.getLogger(CommandMapping.class.getName()); private static final Gson gson = new GsonFireBuilder() - .registerTypeSelector(Command.class, new TypeSelector() { - @Override - public Class getClassForElement(JsonElement jsonElement) { - try { + .registerTypeSelector(Command.class, new TypeSelector() { + @Override + public Class getClassForElement(JsonElement jsonElement) { + try { - String clazz = jsonElement.getAsJsonObject().get(Command.FIELD_FOR_MAPPING).getAsString(); + String clazz = jsonElement.getAsJsonObject().get(Command.FIELD_FOR_MAPPING).getAsString(); - // recursively find correct classes - List list = new ArrayList<>(); - list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) - -> clazz.equals(RegexParser.getFileName(fileName))).scan(CommandMapping.class.getPackage().getName())); + // recursively find correct classes + List list = new ArrayList<>(); + list.addAll(new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) + -> clazz.equals(RegexParser.getFileName(fileName))).scan(CommandMapping.class.getPackage().getName())); - // and use the first one - return (Class) Class.forName(list.get(0).getName()); - } catch (Exception e) { - return null; + // and use the first one + return (Class) Class.forName(list.get(0).getName()); + } catch (Exception e) { + return null; + } } - } - }).createGson(); + }).createGson(); /** * Map object to provided class diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/GenerateUDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/GenerateUDR.java deleted file mode 100644 index c8464da..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/GenerateUDR.java +++ /dev/null @@ -1,45 +0,0 @@ -package ch.icclab.cyclops.consume.command.model; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.consume.command.Command; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * Author: Skoviera - * Created: 29/04/16 - * Description: Command representing request for Events -> UDRs generation - */ -public class GenerateUDR extends Command { - final static Logger logger = LogManager.getLogger(GenerateUDR.class.getName()); - - private String parameter; - - @Override - protected void execute() { - //TODO implement - } - - //====== Getters and Setters - public String getParameter() { - return parameter; - } - public void setParameter(String parameter) { - this.parameter = parameter; - } -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/GenerateCeilometerUDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/ceilometer/GenerateCeilometerUDR.java similarity index 50% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/GenerateCeilometerUDR.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/ceilometer/GenerateCeilometerUDR.java index c4f8018..e9178d5 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/GenerateCeilometerUDR.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/ceilometer/GenerateCeilometerUDR.java @@ -1,17 +1,15 @@ -package ch.icclab.cyclops.consume.command.model; +package ch.icclab.cyclops.consume.command.model.ceilometer; import ch.icclab.cyclops.consume.command.Command; -import ch.icclab.cyclops.consume.data.model.ceilometer.OpenStackCeilometerUDR; +import ch.icclab.cyclops.consume.data.model.ceilometer.OpenStackCeilometerUsageUDR; import ch.icclab.cyclops.consume.data.model.ceilometer.OpenStackCeilometerUsage; import ch.icclab.cyclops.publish.Messenger; import ch.icclab.cyclops.timeseries.BatchPointsContainer; import ch.icclab.cyclops.timeseries.InfluxDBClient; +import ch.icclab.cyclops.timeseries.InfluxDBResponse; import ch.icclab.cyclops.timeseries.QueryBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -41,22 +39,22 @@ public class GenerateCeilometerUDR extends Command { final static Logger logger = LogManager.getLogger(GenerateCeilometerUDR.class.getName()); - private InfluxDBClient influxDBClient = InfluxDBClient.getInstance(); + private InfluxDBClient influxDBClient = new InfluxDBClient(); private Long from; private Long to; @Override - protected void execute() { + protected Object execute() { // Get usage grouped by user and meter where date > hibernate date List usage = getUsage(from, to); // Compute usage data record depending on the meters - List udrs = generateUDR(usage); + List udrs = generateUDR(usage); // Persist the created UDRs BatchPointsContainer container = new BatchPointsContainer(); - for (OpenStackCeilometerUDR udr : udrs) { + for (OpenStackCeilometerUsageUDR udr : udrs) { container.addPoint(udr.toPoint()); } influxDBClient.persistContainer(container); @@ -64,36 +62,48 @@ protected void execute() { // And finally broadcast them Messenger messenger = Messenger.getInstance(); messenger.broadcast(udrs); + return "Ceilometer UDR generated"; } private List getUsage(Long dateFrom, Long dateTo) { - InfluxDBClient influxDBClient = InfluxDBClient.getInstance(); - List usage = influxDBClient.executeQueryAndMapItToClass(new QueryBuilder(OpenStackCeilometerUsage.class.getSimpleName()) - .timeFrom(dateFrom, TimeUnit.SECONDS).timeTo(dateTo, TimeUnit.SECONDS), OpenStackCeilometerUsage.class); - - return usage; + InfluxDBClient influxDBClient = new InfluxDBClient(); + InfluxDBResponse response = influxDBClient.executeQuery(new QueryBuilder(OpenStackCeilometerUsage.class.getSimpleName()) + .timeFrom(dateFrom, TimeUnit.SECONDS).timeTo(dateTo, TimeUnit.SECONDS)); + + try { + return response.getAsListOfType(OpenStackCeilometerUsage.class); + } catch (Exception ignored) { + return new ArrayList<>(); + } } - private List generateUDR(List ceilometerUsage) { - List udrs = new ArrayList<>(); - Map> existingCumulatives = new HashMap<>(); - List gaugeUdrs = new ArrayList<>(); + private List generateUDR(List ceilometerUsage) { + List udrs = new ArrayList<>(); + Map> existingCumulatives = new HashMap<>(); + List gaugeUdrs = new ArrayList<>(); for (OpenStackCeilometerUsage usage : ceilometerUsage) { - OpenStackCeilometerUDR udr; + OpenStackCeilometerUsageUDR udr; if (usage.getMetadata().get("type").equals("cumulative")) { - List lastUsage = influxDBClient.executeQueryAndMapItToClass(new QueryBuilder(OpenStackCeilometerUsage.class.getSimpleName()).where("account", usage.getAccount()).and("meter_name", usage.getMeter_name()).beforeTime(usage.getTime(), TimeUnit.SECONDS).orderDesc().limit(1), OpenStackCeilometerUsage.class); + InfluxDBResponse response = influxDBClient.executeQuery(new QueryBuilder(OpenStackCeilometerUsage.class.getSimpleName()).where("account", usage.getAccount()).and("measurementId", usage.getMeasurementId()).beforeTime(usage.getTime(), TimeUnit.SECONDS).orderDesc().limit(1)); + + try { + List lastUsage = response.getAsListOfType(OpenStackCeilometerUsage.class); - if (lastUsage != null && !lastUsage.isEmpty()) - udr = new OpenStackCeilometerUDR(usage, lastUsage.get(0).getUsage()); - else - // The first import is set as 0 consumption to start the measure from that point - udr = new OpenStackCeilometerUDR(usage, 0.0); + if (lastUsage != null && !lastUsage.isEmpty()) + udr = new OpenStackCeilometerUsageUDR(usage, lastUsage.get(0).getUsage()); + else + // The first import is set as 0 consumption to start the measure from that point + udr = new OpenStackCeilometerUsageUDR(usage, 0.0); + } catch (Exception ignored) { + // if mapping failed for some reason, treat it as 0 consumption too + udr = new OpenStackCeilometerUsageUDR(usage, 0.0); + } // Add the computed Cumulative UDRs to the map udr = processCumulativeUDR(existingCumulatives, udr); } else { // Add the Gauge UDRs to the ArrayList - udr = new OpenStackCeilometerUDR(usage); + udr = new OpenStackCeilometerUsageUDR(usage); } udrs.add(udr); } @@ -101,18 +111,18 @@ private List generateUDR(List return udrs; } - private OpenStackCeilometerUDR processCumulativeUDR(Map> existingCumulatives, OpenStackCeilometerUDR udr) { - if (!existingCumulatives.containsKey(udr.getMeter_name())) { - List value = new ArrayList<>(); + private OpenStackCeilometerUsageUDR processCumulativeUDR(Map> existingCumulatives, OpenStackCeilometerUsageUDR udr) { + if (!existingCumulatives.containsKey(udr.getMeasurementId())) { + List value = new ArrayList<>(); value.add(udr); - existingCumulatives.put(udr.getMeter_name(), value); + existingCumulatives.put(udr.getMeasurementId(), value); } else { - List values = existingCumulatives.get(udr.getMeter_name()); - for (OpenStackCeilometerUDR value : values) { + List values = existingCumulatives.get(udr.getMeasurementId()); + for (OpenStackCeilometerUsageUDR value : values) { udr.setUsage(udr.getUsage() - value.getUsage()); } values.add(udr); - existingCumulatives.put(udr.getMeter_name(), values); + existingCumulatives.put(udr.getMeasurementId(), values); } return udr; } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/GenerateUDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/GenerateUDR.java new file mode 100644 index 0000000..4c35ff7 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/GenerateUDR.java @@ -0,0 +1,208 @@ +package ch.icclab.cyclops.consume.command.model.generic; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.consume.command.Command; +import ch.icclab.cyclops.consume.command.model.generic.model.UDR; +import ch.icclab.cyclops.consume.command.model.generic.model.UsageData; +import ch.icclab.cyclops.publish.Messenger; +import ch.icclab.cyclops.timeseries.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.atteo.evo.inflector.English; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * Author: Skoviera + * Created: 29/04/16 + * Description: Command representing request for Events -> UDRs generation + */ +public class GenerateUDR extends Command { + final static Logger logger = LogManager.getLogger(GenerateUDR.class.getName()); + + private Long from; + private Long to; + + // notify subscribed parties + private boolean broadcast; + + // route message via dispatch + private boolean dispatch; + + // output UDRs synchronously + private boolean output; + + public static class Measurement { + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + /** + * Validate request and create UDR records for individual accounts + * @return string confirmation message + */ + @Override + protected Object execute() { + + // sanity checks first + if (from == null || to == null || from < 0l || to <= from) { + return "[ERROR] invalid FROM and TO"; + } + + // get list of measurements + List measurements = getListOfMeasurements(); + if (measurements != null && !measurements.isEmpty()) { + + // flat container + List usageData = new ArrayList<>(); + + // query all measurements and get Usage Data + measurements.stream().forEach(measurement -> usageData.addAll(queryMeasurement(measurement))); + + if (!usageData.isEmpty()) { + // downsample by summing up streams and generate UDR records + List UDRs = downsampleAndGenerate(usageData); + + // persist them to database and output status code + persistUDRs(UDRs); + + // broadcast if desired + if (broadcast) { + Messenger.getInstance().broadcast(UDRs); + } + + // dispatch if desired + if (dispatch) { + Messenger.getInstance().publish(UDRs, getClass().getSimpleName()); + } + + // either return created UDRs or output a message + return (output)? UDRs : String.format("%d %s generated", UDRs.size(), English.plural("UDR", UDRs.size())); + + } else { + return "No data"; + } + + } else { + return "No data"; + } + } + + /** + * Get list of measurements + * @return list of null + */ + private List getListOfMeasurements(){ + try { + // prepare and execute query + QueryBuilder builder = QueryBuilder.getMeasurementsQuery(); + InfluxDBResponse result = new InfluxDBClient().executeQuery(builder); + + // map response to Measurement class + List measurements = result.getAsListOfType(Measurement.class); + + // get it as a list of Strings and filter out UDR measurement + return measurements.stream().map(Measurement::getName).filter(name -> !name.equals(UDR.class.getSimpleName())).collect(Collectors.toList()); + } catch (Exception e) { + return null; + } + } + + + /** + * Query database and list usage data for specified measurement + * @param measurement to query + * @return UsageData list + */ + private List queryMeasurement(String measurement) { + try { + // construct and execute query + QueryBuilder query = new QueryBuilder(measurement).timeFrom(from, TimeUnit.SECONDS).timeTo(to, TimeUnit.SECONDS); + InfluxDBResponse response = new InfluxDBClient().executeQuery(query); + + // parse as list of UsageData + return response.getAsListOfType(UsageData.class); + } catch (Exception ignored) { + return new ArrayList<>(); + } + } + + /** + * Go over list of Usage Data, downsample it and generate UDRs + * @param data to be processed + * @return list of UDRs + */ + private List downsampleAndGenerate(List data) { + + Map accounts = new HashMap<>(); + + // go over individual items and downsample them + for (UsageData item : data) { + + // container for account data + List accountData = accounts.getOrDefault(item.getAccount(), new ArrayList<>()); + + // find index of the item + Integer index = accountData.indexOf(item); + if (index >= 0) { + // update original record + accountData.get(index).addToUsage(item.getUsage()); + } else { + // add record to the list + accountData.add(item); + } + + // put it back + accounts.put(item.getAccount(), accountData); + } + + // now that we are done, let's create UDR records + List UDRs = new ArrayList<>(); + for (Map.Entry entry: accounts.entrySet()) { + UDR udr = new UDR(entry.getKey(), from, to); + udr.setData(entry.getValue()); + UDRs.add(udr); + } + + return UDRs; + } + + /** + * Persist UDR records into database (synchronously) + * @param UDRs to be persisted + */ + private void persistUDRs(List UDRs) { + // prepare container + BatchPointsContainer container = new BatchPointsContainer(); + + // fill container with individual points + UDRs.stream().forEach(udr -> container.addPoint(GenerateDBPoint.fromObjectWithTimeAndTags(udr, UDR.getTimeFieldName(), UDR.getTimeUnit(), UDR.getTagNames()))); + + // persist container + new InfluxDBClient().persistContainer(container); + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/UDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/UDR.java new file mode 100644 index 0000000..7af9027 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/UDR.java @@ -0,0 +1,89 @@ +package ch.icclab.cyclops.consume.command.model.generic.model; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Author: Skoviera + * Created: 06/09/16 + * Description: UDR output + */ +public class UDR { + private String _class = getClass().getSimpleName(); + private String account; + private Long time; + private Long from; + private Long to; + private List data; + + public static List getTagNames() { + return Collections.singletonList("account"); + } + + public static String getTimeFieldName() { + return "time"; + } + + public static TimeUnit getTimeUnit() { + return TimeUnit.SECONDS; + } + + public UDR(String account, Long from, Long to) { + this.time = System.currentTimeMillis() / 1000; + this.account = account; + this.from = from; + this.to = to; + } + + public String getAccount() { + return account; + } + public void setAccount(String account) { + this.account = account; + } + + public Long getTime() { + return time; + } + public void setTime(Long time) { + this.time = time; + } + + public List getData() { + return data; + } + public void setData(List data) { + this.data = data; + } + + public Long getFrom() { + return from; + } + public void setFrom(Long from) { + this.from = from; + } + + public Long getTo() { + return to; + } + public void setTo(Long to) { + this.to = to; + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/UsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/UsageData.java new file mode 100644 index 0000000..c2fe753 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/command/model/generic/model/UsageData.java @@ -0,0 +1,88 @@ +package ch.icclab.cyclops.consume.command.model.generic.model; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.base.Objects; + +import java.util.Map; + +/** + * Author: Skoviera + * Created: 06/09/16 + * Description: Usage record coming from the database + */ +public class UsageData { + private String _class; + private String account; + private Double usage; + private Object unit; + private Map metadata; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UsageData usageData = (UsageData) o; + return com.google.common.base.Objects.equal(_class, usageData._class) && + Objects.equal(account, usageData.account) && + Objects.equal(unit, usageData.unit) && + Objects.equal(metadata, usageData.metadata); + } + + @Override + public int hashCode() { + return Objects.hashCode(_class, account, unit, metadata); + } + + public String get_class() { + return _class; + } + public void set_class(String _class) { + this._class = _class; + } + + public String getAccount() { + return account; + } + public void setAccount(String account) { + this.account = account; + } + + public Double getUsage() { + return usage; + } + public void setUsage(Double usage) { + this.usage = usage; + } + public void addToUsage(Double usage) { + this.usage += usage; + } + + public Object getUnit() { + return unit; + } + public void setUnit(Object unit) { + this.unit = unit; + } + + public Map getMetadata() { + return metadata; + } + public void setMetadata(Map metadata) { + this.metadata = metadata; + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java index 4bb2e74..584efd3 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/DataConsumer.java @@ -1,24 +1,10 @@ package ch.icclab.cyclops.consume.data; import ch.icclab.cyclops.consume.AbstractConsumer; -import ch.icclab.cyclops.load.Loader; +import ch.icclab.cyclops.executor.TaskExecutor; import ch.icclab.cyclops.load.model.PublisherCredentials; -import ch.icclab.cyclops.publish.Messenger; -import ch.icclab.cyclops.timeseries.BatchPointsContainer; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.RemoveNullValues; -import ch.icclab.cyclops.util.RegexParser; -import ch.icclab.cyclops.util.loggers.DataLogger; -import com.github.wnameless.json.flattener.JsonFlattener; -import com.google.gson.Gson; -import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.influxdb.dto.Point; - -import java.math.BigDecimal; -import java.util.*; -import java.util.concurrent.TimeUnit; /** * Author: Skoviera @@ -28,371 +14,19 @@ public class DataConsumer extends AbstractConsumer { final static Logger logger = LogManager.getLogger(DataConsumer.class.getName()); - private static InfluxDBClient influxDBClient = InfluxDBClient.getInstance(); - private static Messenger messenger = Messenger.getInstance(); - - public static TimeUnit TIME_UNIT = TimeUnit.SECONDS; - private PublisherCredentials publisherSettings; private String defaultMeasurementName; + private TaskExecutor executor; - private Long validRecords; - - private BatchPointsContainer container; - private List broadcast; - private Map> dispatch; public DataConsumer(String name, PublisherCredentials settings) { this.defaultMeasurementName = name; this.publisherSettings = settings; + this.executor = TaskExecutor.getInstance(); } @Override public void consume(String content) { - initialise(); - - try { - // try to map it as array - List array = new Gson().fromJson(content, List.class); - - // iterate over object entries - array.forEach(this::processDataFrame); - - } catch (Exception ignored) { - // this means it was not an array to begin with, just a simple object - processDataFrame(content); - } - - finalise(); - } - - public Long getNumberOfValidRecords() { - return validRecords; - } - - private void initialise() { - validRecords = 0l; - container = new BatchPointsContainer(); - broadcast = new ArrayList<>(); - dispatch = new HashMap<>(); - } - - private void finalise() { - // persist points that were created - influxDBClient.persistContainer(container); - - // broadcast - if (broadcast != null && !broadcast.isEmpty()) { - if (broadcast.size() == 1) { - // broadcast it as value - messenger.broadcast(broadcast.get(0)); - } else { - // broadcast it as array - messenger.broadcast(broadcast); - } - } - - // dispatch - if (dispatch != null && !dispatch.isEmpty()) { - for (Map.Entry entry: dispatch.entrySet()) { - List list = (List) entry.getValue(); - - if (list.size() == 1) { - // publish it as value - messenger.publish(list.get(0), (String) entry.getKey()); - } else { - // publish it as array - messenger.publish(list, (String) entry.getKey()); - } - } - } - } - - /** - * Process data frame as OBJECT - * @param obj to be processed - */ - private void processDataFrame(Map obj) { - storeAndPublish(obj); - } - - /** - * Process data frame as STRING - * @param str to be processed - */ - private void processDataFrame(String str) { - try { - Map obj = new Gson().fromJson(str, Map.class); - - storeAndPublish(obj); - } catch (Exception ignored) { - // if incoming JSON is not valid we are simply skipping it - } - } - - /** - * Process, store and publish incoming object - * @param obj representation - */ - private void storeAndPublish(Map obj) { - try { - // now parse it and create a database Point - ConsumedData data = processDataAndGetPoint(obj); - - // and finally persist it - container.addPoint(data.getPoint()); - - // publish or broadcast if desired - if (data.shouldPublish()) { - - // reapply class definition - obj.put(DataMapping.FIELD_FOR_MAPPING, data.getClazz()); - - // where to publish - if (data.doNotBroadcastButRoute()) { - String clazz = data.getClazz(); - - // add to internal dispatch structure - if (dispatch.containsKey(clazz)) { - // we need to fetch the list - List list = dispatch.get(clazz); - list.add(obj); - } else { - // we need to create a new one - List list = new ArrayList<>(); - list.add(obj); - - dispatch.put(clazz, list); - } - - } else { - broadcast.add(obj); - } - } - - validRecords += 1; - - } catch (Exception notValidJson) { - DataLogger.log(String.format("Received event/measurement is not a valid JSON: %s", notValidJson.getMessage())); - } - } - - /** - * Data holder for consumed data including database point and mapping guidelines - */ - private class ConsumedData { - private String clazz; - private Point.Builder builder; - private Boolean shouldPublish; - private Boolean doNotBroadcastButRoute; - - public ConsumedData(String clazz, Point.Builder builder, Boolean shouldPublish, Boolean doNotBroadcastButRoute) { - this.clazz = clazz; - this.builder = builder; - this.shouldPublish = shouldPublish; - this.doNotBroadcastButRoute = doNotBroadcastButRoute; - } - - public ConsumedData(String clazz, Point.Builder builder, DataMapping guideline) { - this.clazz = clazz; - this.builder = builder; - this.shouldPublish = guideline.shouldPublish(); - this.doNotBroadcastButRoute = guideline.doNotBroadcastButRoute(); - } - - public String getClazz() { - return clazz; - } - public Point.Builder getPoint() { - return builder; - } - public Boolean shouldPublish() { - return shouldPublish; - } - public Boolean doNotBroadcastButRoute() { - return doNotBroadcastButRoute; - } - } - - /** - * Process and create database Point - * @param map to be processed - * @return point - */ - private ConsumedData processDataAndGetPoint(Map map) { - try { - // check whether type field is present - String clazz = (String) map.get(DataMapping.FIELD_FOR_MAPPING); - - // find corresponding classes - Set set = new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) - -> clazz.equals(RegexParser.getFileName(fileName))).scan(DataConsumer.class.getPackage().getName()); - - // move to the first element - Optional first = set.stream().findFirst(); - - if (first.isPresent()) { - // access object based on provided class definition - DataMapping guideline = (DataMapping) Class.forName(first.get().getName()).newInstance(); - - DataLogger.log(String.format("Event/measurement received with \"%s\" guidelines", clazz)); - - // by default we will work with the original map - Map processed = map; - try { - // let's call user specified pre-processing method - Map preProcessing = guideline.preProcess(map); - - // assign new one if it's valid - if (preProcessing != null && !preProcessing.isEmpty()) { - processed = preProcessing; - } - } catch (Exception ignored) { - } - - // we will be storing flattened version of the map - Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(processed)); - - // remove type from original HashMap - flat.remove(DataMapping.FIELD_FOR_MAPPING); - - // extract tags from flattened json - Map tags = extractTags(flat, guideline.getTagNames()); - - // parse time field if it was provided - String timeField = guideline.getTimeField(); - Long timeStamp = (timeField != null && flat.containsKey(timeField))? getTimeStamp(flat.get(timeField)): null; - if (timeStamp != null && timeStamp >= 0) flat.remove(timeField); - - // determine correct time unit - TimeUnit unit = guideline.getTimeUnit(); - if (unit == null) { - unit = TIME_UNIT; - } - // finally construct database point - Point.Builder builder = constructPoint(clazz, timeStamp, unit, flat, tags); - return new ConsumedData(clazz, builder, guideline); - } else { - // guideline was not present or wasn't valid - throw new Exception(); - } - - } catch (Exception withoutGuidelines){ - // mapping failed, guideline was not provided, use default - DataLogger.log("Event/measurement received without valid guidelines"); - - Point.Builder builder; - - Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(map)); - - // even though we don't have mapping template, we still want to have proper measurement name if TYPE is present - if (flat.containsKey(DataMapping.FIELD_FOR_MAPPING)){ - - // get that name - String measurement = (String) flat.get(DataMapping.FIELD_FOR_MAPPING); - - // remove it from flat - flat.remove(DataMapping.FIELD_FOR_MAPPING); - - // construct point - builder = constructPoint(measurement, flat); - } else { - builder = constructPoint(defaultMeasurementName, flat); - } - - // determine publisher preferences - Boolean shouldPublish = publisherSettings.getPublisherIncludeAlsoUnknown(); - Boolean doNotRouteButBroadcast = publisherSettings.getPublisherByDefaultDispatchInsteadOfBroadcast(); - String clazz = Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement(); - - return new ConsumedData(clazz, builder, shouldPublish, doNotRouteButBroadcast); - } - } - - /** - * Construct database Point - * @param measurementName to be used - * @param fields as map - * @return point builder - */ - private Point.Builder constructPoint(String measurementName, Map fields) { - return Point.measurement(measurementName).fields(RemoveNullValues.fromMap(fields)); - - } - - /** - * Construct database Point - * @param measurementName to be used - * @param timeStamp to be marked - * @param unit of time to be used - * @param fields as map - * @param tags as map - * @return point builder - */ - private Point.Builder constructPoint(String measurementName, Long timeStamp, TimeUnit unit, Map fields, Map tags) { - Point.Builder builder = Point.measurement(measurementName); - - // apply time stamp with time unit - if (timeStamp != null && timeStamp >= 0 && unit != null) { - builder.time(timeStamp, unit); - } - - // apply specified tags - if (tags != null && !tags.isEmpty()) { - builder.tag(RemoveNullValues.fromMap(tags)); - } - - // and finally add fields - builder.fields(RemoveNullValues.fromMap(fields)); - - return builder; - } - - /** - * Move tags from content into variable based on provided list of tags - * @param content to be extracted - * @param tagList definition - * @return extracted tags or null - */ - private Map extractTags(Map content, List tagList) { - if (tagList != null) { - Map tags = new HashMap<>(); - - // iterate over list of tags and move them from the original content - for (String tag: tagList) { - tags.put(tag, (String) content.get(tag)); - content.remove(tag); - } - - return tags; - } else { - return null; - } - } - - /** - * Parse long from unknown type - * @param unknown object - * @return Long or null - */ - private Long getTimeStamp(Object unknown) { - try { - BigDecimal dec = (BigDecimal) unknown; - return dec.longValue(); - } catch (Exception a) { - try { - return Long.parseLong((String) unknown); - } catch (Exception b) { - try { - return (Long) unknown; - } catch (Exception c) { - try { - return Long.valueOf(unknown.toString()); - } catch (Exception d) { - return null; - } - } - } - } + executor.addTask(new DataProcess(content, publisherSettings, defaultMeasurementName)); } } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java new file mode 100644 index 0000000..0668732 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/DataProcess.java @@ -0,0 +1,373 @@ +package ch.icclab.cyclops.consume.data; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.executor.TaskExecutor; +import ch.icclab.cyclops.load.Loader; +import ch.icclab.cyclops.load.model.PublisherCredentials; +import ch.icclab.cyclops.publish.Messenger; +import ch.icclab.cyclops.timeseries.*; +import ch.icclab.cyclops.util.RegexParser; +import ch.icclab.cyclops.util.loggers.DataLogger; +import com.github.wnameless.json.flattener.JsonFlattener; +import com.google.gson.Gson; +import com.metapossum.utils.scanner.reflect.ClassesInPackageScanner; +import org.influxdb.dto.Point; + +import java.math.BigDecimal; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Process data frame in runnable manner + */ +public class DataProcess implements Runnable { + + private String content; + + private static Messenger messenger = Messenger.getInstance(); + + private PublisherCredentials publisherSettings; + private String defaultMeasurementName; + + private List broadcast; + private BatchPointsContainer container; + private Map> dispatch; + + public static TimeUnit TIME_UNIT = TimeUnit.SECONDS; + + public DataProcess(String content, PublisherCredentials publisher, String measurement) { + this.publisherSettings = publisher; + this.defaultMeasurementName = measurement; + this.container = new BatchPointsContainer(); + this.broadcast = new ArrayList<>(); + this.dispatch = new HashMap<>(); + this.content = content; + } + + @Override + public void run() { + try { + // try to map it as array + List array = new Gson().fromJson(content, List.class); + + // process individual items (cannot be in parallel as we would quickly start thread-context switching) + array.stream().forEach(this::processDataFrame); + + } catch (Exception ignored) { + // this means it was not an array to begin with, just a simple object + processDataFrame(content); + } + + finalise(); + } + + private void finalise() { + + // schedule tasks + TaskExecutor executor = TaskExecutor.getInstance(); + + // persist points + if (container.size() == 1) { + // when only one point in container use shared InfluxDB session for delaying writes + executor.addTask(() -> SharedInfluxDBSession.getSession().persistSinglePoint(container.getFirstPoint())); + } else if (container.size() > 1) { + // we did batch persisting manually so we can store it immediately + executor.addTask(() -> {new InfluxDBClient().persistContainer(container);}); + } + + // broadcast + if (broadcast != null && !broadcast.isEmpty()) { + if (broadcast.size() == 1) { + // broadcast it as value + executor.addTask(() -> {messenger.broadcast(broadcast.get(0));}); + } else { + // broadcast it as array + executor.addTask(() -> {messenger.broadcast(broadcast);}); + } + } + + // dispatch + if (dispatch != null && !dispatch.isEmpty()) { + for (Map.Entry entry: dispatch.entrySet()) { + List list = (List) entry.getValue(); + + if (list.size() == 1) { + // publish it as value + executor.addTask(() -> {messenger.publish(list.get(0), (String) entry.getKey());}); + } else { + // publish it as array + executor.addTask(() -> {messenger.publish(list, (String) entry.getKey());}); + } + } + } + } + + /** + * Process data frame as OBJECT + * @param obj to be processed + */ + private void processDataFrame(Map obj) { + storeAndPublish(obj); + } + + /** + * Process data frame as STRING + * @param str to be processed + */ + private void processDataFrame(String str) { + try { + Map obj = new Gson().fromJson(str, Map.class); + + storeAndPublish(obj); + } catch (Exception ignored) { + // if incoming JSON is not valid we are simply skipping it + } + } + + /** + * Process, store and publish incoming object + * @param obj representation + */ + private void storeAndPublish(Map obj) { + try { + // now parse it and create a database Point + ConsumedData data = processDataAndGetPoint(obj); + + // and finally persist it + container.addPoint(data.getPoint()); + + // publish or broadcast if desired + if (data.shouldPublish()) { + + // where to publish + if (data.doNotBroadcastButRoute()) { + String clazz = data.getClazz(); + + // add to internal dispatch structure + if (dispatch.containsKey(clazz)) { + // we need to fetch the list + List list = dispatch.get(clazz); + list.add(obj); + } else { + // we need to create a new one + List list = new ArrayList<>(); + list.add(obj); + + dispatch.put(clazz, list); + } + + } else { + broadcast.add(obj); + } + } + + } catch (Exception notValidJson) { + DataLogger.log(String.format("Received event/measurement is not a valid JSON: %s", notValidJson.getMessage())); + } + } + + /** + * Data holder for consumed data including database point and mapping guidelines + */ + private class ConsumedData { + private String clazz; + private Point.Builder builder; + private Boolean shouldPublish; + private Boolean doNotBroadcastButRoute; + + public ConsumedData(String clazz, Point.Builder builder, Boolean shouldPublish, Boolean doNotBroadcastButRoute) { + this.clazz = clazz; + this.builder = builder; + this.shouldPublish = shouldPublish; + this.doNotBroadcastButRoute = doNotBroadcastButRoute; + } + + public ConsumedData(String clazz, Point.Builder builder, DataMapping guideline) { + this.clazz = clazz; + this.builder = builder; + this.shouldPublish = guideline.shouldPublish(); + this.doNotBroadcastButRoute = guideline.doNotBroadcastButRoute(); + } + + public String getClazz() { + return clazz; + } + public Point.Builder getPoint() { + return builder; + } + public Boolean shouldPublish() { + return shouldPublish; + } + public Boolean doNotBroadcastButRoute() { + return doNotBroadcastButRoute; + } + } + + /** + * Process and create database Point + * @param map to be processed + * @return point + */ + private ConsumedData processDataAndGetPoint(Map map) { + try { + // check whether type field is present + String clazz = (String) map.get(DataMapping.FIELD_FOR_MAPPING); + + // find corresponding classes + Set set = new ClassesInPackageScanner().setResourceNameFilter((packageName, fileName) + -> clazz.equals(RegexParser.getFileName(fileName))).scan(DataConsumer.class.getPackage().getName()); + + // move to the first element + Optional first = set.stream().findFirst(); + + if (first.isPresent()) { + // access object based on provided class definition + DataMapping guideline = (DataMapping) Class.forName(first.get().getName()).newInstance(); + + DataLogger.log(String.format("Event/measurement received with \"%s\" guidelines", clazz)); + + // by default we will work with the original map + Map processed = map; + try { + // let's call user specified pre-processing method + Map preProcessing = guideline.preProcess(map); + + // assign new one if it's valid + if (preProcessing != null && !preProcessing.isEmpty()) { + processed = preProcessing; + } + } catch (Exception ignored) { + } + + // we will be storing flattened version of the map + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(processed)); + + // extract tags from flattened json + Map tags = extractTags(flat, guideline.getTagNames()); + + // parse time field if it was provided + String timeField = guideline.getTimeField(); + Long timeStamp = (timeField != null && flat.containsKey(timeField))? TimeStamp.cast(flat.get(timeField)): null; + if (timeStamp != null && timeStamp >= 0) flat.remove(timeField); + + // determine correct time unit + TimeUnit unit = guideline.getTimeUnit(); + if (unit == null) { + unit = TIME_UNIT; + } + // finally construct database point + Point.Builder builder = constructPoint(clazz, timeStamp, unit, flat, tags); + return new ConsumedData(clazz, builder, guideline); + } else { + // guideline was not present or wasn't valid + throw new Exception(); + } + + } catch (Exception withoutGuidelines){ + // mapping failed, guideline was not provided, use default + DataLogger.log("Event/measurement received without valid guidelines"); + + Point.Builder builder; + + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(map)); + + // even though we don't have mapping template, we still want to have proper measurement name if TYPE is present + if (flat.containsKey(DataMapping.FIELD_FOR_MAPPING)){ + + // get that name + String measurement = (String) flat.get(DataMapping.FIELD_FOR_MAPPING); + + // construct point + builder = constructPoint(measurement, flat); + } else { + builder = constructPoint(defaultMeasurementName, flat); + } + + // determine publisher preferences + Boolean shouldPublish = publisherSettings.getPublisherIncludeAlsoUnknown(); + Boolean doNotRouteButBroadcast = publisherSettings.getPublisherByDefaultDispatchInsteadOfBroadcast(); + String clazz = Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement(); + + return new ConsumedData(clazz, builder, shouldPublish, doNotRouteButBroadcast); + } + } + + /** + * Construct database Point + * @param measurementName to be used + * @param fields as map + * @return point builder + */ + private Point.Builder constructPoint(String measurementName, Map fields) { + return Point.measurement(measurementName).time(System.currentTimeMillis(), TimeUnit.MILLISECONDS).fields(RemoveNullValues.fromMap(fields)); + } + + /** + * Construct database Point + * @param measurementName to be used + * @param timeStamp to be marked + * @param unit of time to be used + * @param fields as map + * @param tags as map + * @return point builder + */ + private Point.Builder constructPoint(String measurementName, Long timeStamp, TimeUnit unit, Map fields, Map tags) { + Point.Builder builder = Point.measurement(measurementName); + + // apply time stamp with time unit + if (timeStamp != null && timeStamp >= 0 && unit != null) { + builder.time(timeStamp, unit); + } else { + builder.time(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + } + + // apply specified tags + if (tags != null && !tags.isEmpty()) { + builder.tag(RemoveNullValues.fromMap(tags)); + } + + // and finally add fields + builder.fields(RemoveNullValues.fromMap(fields)); + + return builder; + } + + /** + * Move tags from content into variable based on provided list of tags + * @param content to be extracted + * @param tagList definition + * @return extracted tags or null + */ + private Map extractTags(Map content, List tagList) { + if (tagList != null) { + Map tags = new HashMap<>(); + + // iterate over list of tags and move them from the original content + for (String tag: tagList) { + tags.put(tag, (String) content.get(tag)); + content.remove(tag); + } + + return tags; + } else { + return null; + } + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ExternalUDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometer.java similarity index 87% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ExternalUDR.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometer.java index 77e7614..78ca4cc 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ExternalUDR.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometer.java @@ -1,4 +1,4 @@ -package ch.icclab.cyclops.consume.data.model; +package ch.icclab.cyclops.consume.data.model.ceilometer; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -25,10 +25,10 @@ /** * Author: Skoviera - * Created: 13/05/16 - * Description: ExternalUDR measurement definition + * Created: 09/09/16 + * Description: Data mapping for OpenStack Ceilometer */ -public class ExternalUDR implements DataMapping { +public class OpenStackCeilometer implements DataMapping{ @Override public String getTimeField() { return "time"; @@ -53,7 +53,7 @@ public Map preProcess(Map original) { @Override public Boolean shouldPublish() { - return true; + return false; } @Override diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/CloudStackCDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerCPU.java similarity index 78% rename from core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/CloudStackCDR.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerCPU.java index a0271a6..9fce891 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/CloudStackCDR.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerCPU.java @@ -1,4 +1,4 @@ -package ch.icclab.cyclops.consume.data.model; +package ch.icclab.cyclops.consume.data.model.ceilometer; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -18,8 +18,8 @@ /** * Author: Skoviera - * Created: 26/05/16 - * Description: CloudStack CDR specification + * Created: 09/09/16 + * Description: Data mapping for Open Stack Ceilometer CPU */ -public class CloudStackCDR extends GenericCDR{ +public class OpenStackCeilometerCPU extends OpenStackCeilometer{ } diff --git a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/ExternalCDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerRAM.java similarity index 78% rename from core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/ExternalCDR.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerRAM.java index d08c136..589bccc 100644 --- a/core/rc/cdr/src/main/java/ch/icclab/cyclops/consume/data/model/ExternalCDR.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerRAM.java @@ -1,4 +1,4 @@ -package ch.icclab.cyclops.consume.data.model; +package ch.icclab.cyclops.consume.data.model.ceilometer; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -18,8 +18,8 @@ /** * Author: Skoviera - * Created: 26/05/16 - * Description: External CDR specification + * Created: 09/09/16 + * Description: Data mapping for Open Stack Ceilometer RAM */ -public class ExternalCDR extends GenericCDR{ +public class OpenStackCeilometerRAM extends OpenStackCeilometer{ } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUsage.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUsage.java index 1cbdb16..cf712c4 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUsage.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUsage.java @@ -31,7 +31,8 @@ public class OpenStackCeilometerUsage implements DataMapping{ private long time; private Double usage; private String account; - private String meter_name; + private String measurementId; + private String unit; private Map metadata; @Override @@ -48,7 +49,7 @@ public TimeUnit getTimeUnit() { public List getTagNames() { List list = new ArrayList<>(); list.add("account"); - list.add("meter_name"); + list.add("measurementId"); return list; } @@ -99,14 +100,6 @@ public void setAccount(String account) { this.account = account; } - public String getMeter_name() { - return meter_name; - } - - public void setMeter_name(String meter_name) { - this.meter_name = meter_name; - } - public Map getMetadata() { return metadata; } @@ -114,4 +107,20 @@ public Map getMetadata() { public void setMetadata(Map metadata) { this.metadata = metadata; } + + public String getUnit() { + return unit; + } + + public void setUnit(String unit) { + this.unit = unit; + } + + public String getMeasurementId() { + return measurementId; + } + + public void setMeasurementId(String measurementId) { + this.measurementId = measurementId; + } } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUsageUDR.java similarity index 75% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUDR.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUsageUDR.java index 625e333..31e1598 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUDR.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/ceilometer/OpenStackCeilometerUsageUDR.java @@ -28,32 +28,36 @@ * Created by Manu Perez on 11/07/16. */ -public class OpenStackCeilometerUDR { +public class OpenStackCeilometerUsageUDR { private String _class = this.getClass().getSimpleName(); private long time; private String account; private double usage; - private String meter_name; - private String type; + private String measurementId; +// Type is removed in order to keep generality +// private String type; private String unit; + private Map metadata; - public OpenStackCeilometerUDR(OpenStackCeilometerUsage usageData) { + public OpenStackCeilometerUsageUDR(OpenStackCeilometerUsage usageData) { // Constructor for Gauge, Delta Meters or first cumulative measurement this.time = usageData.getTime(); this.account = usageData.getAccount(); - this.meter_name = usageData.getMeter_name(); + this.measurementId = usageData.getMeasurementId(); this.usage = usageData.getUsage(); - this.type = (String) usageData.getMetadata().get("type"); - this.unit = (String) usageData.getMetadata().get("unit"); +// this.type = (String) usageData.getMetadata().get("type"); + this.unit = usageData.getUnit(); + this.metadata = usageData.getMetadata(); } - public OpenStackCeilometerUDR(OpenStackCeilometerUsage usageData, Double usage) { + public OpenStackCeilometerUsageUDR(OpenStackCeilometerUsage usageData, Double usage) { // Constructor for Cumulative Meters this.time = usageData.getTime(); this.account = usageData.getAccount(); - this.meter_name = usageData.getMeter_name(); - this.type = (String) usageData.getMetadata().get("type"); - this.unit = (String) usageData.getMetadata().get("unit"); + this.measurementId = usageData.getMeasurementId(); +// this.type = (String) usageData.getMetadata().get("type"); + this.unit = usageData.getUnit(); + this.metadata = usageData.getMetadata(); if (usageData.getUsage() == usage || usage == 0) { // In case that the Usage is the same than the last Ceilometer Usage, the consumption has been 0 @@ -90,7 +94,7 @@ private Map getTags() { Map map = new HashMap(); map.put("account", account); - map.put("meter_name", meter_name); + map.put("measurementId", measurementId); return map; } @@ -103,7 +107,8 @@ private Map getTags() { private Map getFields() { Map map = new HashMap(); map.put("usage", usage); - map.put("type", type); +// map.put("type", type); + map.put("metadata", metadata); map.put("unit", unit); return map; @@ -160,19 +165,19 @@ public void setUsage(double usage) { this.usage = usage; } - public String getMeter_name() { - return meter_name; + public String getMeasurementId() { + return measurementId; } - public void setMeter_name(String meter_name) { - this.meter_name = meter_name; + public void setMeasurementId(String measurementId) { + this.measurementId = measurementId; } - public String getType() { - return type; + public Map getMetadata() { + return metadata; } - public void setType(String type) { - this.type = type; + public void setMetadata(Map metadata) { + this.metadata = metadata; } } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVPNUserUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackIP.java similarity index 83% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVPNUserUsageData.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackIP.java index 1afadf9..b4f4155 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVPNUserUsageData.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackIP.java @@ -18,8 +18,8 @@ /** * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack VPN User Usage Data + * Created: 12/09/16 + * Description: Data mapping for IP */ -public class CloudStackVPNUserUsageData extends CloudStackUsageData { +public class CloudStackIP extends CloudStackUsageData { } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackSnapshotUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackISO.java similarity index 83% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackSnapshotUsageData.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackISO.java index 5862775..1a02533 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackSnapshotUsageData.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackISO.java @@ -18,8 +18,8 @@ /** * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack Snapshot Usage Data + * Created: 12/09/16 + * Description: Data mapping for ISO */ -public class CloudStackSnapshotUsageData extends CloudStackUsageData { +public class CloudStackISO extends CloudStackUsageData { } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackLoadBalancer.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackLoadBalancer.java new file mode 100644 index 0000000..7705833 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackLoadBalancer.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for Load Balancer + */ +public class CloudStackLoadBalancer extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVolumeUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkOffering.java similarity index 83% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVolumeUsageData.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkOffering.java index b3fcf40..5bd515b 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVolumeUsageData.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkOffering.java @@ -18,8 +18,8 @@ /** * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack Volume Usage Data + * Created: 12/09/16 + * Description: Data mapping for Network Offering */ -public class CloudStackVolumeUsageData extends CloudStackUsageData { +public class CloudStackNetworkOffering extends CloudStackUsageData { } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkReceived.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkReceived.java new file mode 100644 index 0000000..086d890 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkReceived.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for Network Received + */ +public class CloudStackNetworkReceived extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackIPUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkSent.java similarity index 84% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackIPUsageData.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkSent.java index e2862ee..abe37d1 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackIPUsageData.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkSent.java @@ -18,8 +18,8 @@ /** * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack IP Usage Data + * Created: 12/09/16 + * Description: Data mapping for Network Sent */ -public class CloudStackIPUsageData extends CloudStackUsageData { +public class CloudStackNetworkSent extends CloudStackUsageData { } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkUsageData.java deleted file mode 100644 index dd57753..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackNetworkUsageData.java +++ /dev/null @@ -1,25 +0,0 @@ -package ch.icclab.cyclops.consume.data.model.cloudstack; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/** - * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack Network Usage Data (both Sent and Received) - */ -public class CloudStackNetworkUsageData extends CloudStackUsageData { -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackPolicyOrRuleUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackPolicyOrRuleUsageData.java deleted file mode 100644 index 75c2d4b..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackPolicyOrRuleUsageData.java +++ /dev/null @@ -1,25 +0,0 @@ -package ch.icclab.cyclops.consume.data.model.cloudstack; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/** - * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack Load Balancer Policy and Port Forwarding Rule data - */ -public class CloudStackPolicyOrRuleUsageData extends CloudStackUsageData { -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackPortForwarding.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackPortForwarding.java new file mode 100644 index 0000000..174c814 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackPortForwarding.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for Port Forwarding + */ +public class CloudStackPortForwarding extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackSnapshot.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackSnapshot.java new file mode 100644 index 0000000..a9cf15c --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackSnapshot.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for Snapshot + */ +public class CloudStackSnapshot extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackTemplate.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackTemplate.java new file mode 100644 index 0000000..0c15aa9 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackTemplate.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for Template + */ +public class CloudStackTemplate extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackTemplateAndIsoUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackTemplateAndIsoUsageData.java deleted file mode 100644 index 191a634..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackTemplateAndIsoUsageData.java +++ /dev/null @@ -1,25 +0,0 @@ -package ch.icclab.cyclops.consume.data.model.cloudstack; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/** - * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack Template and ISO Usage Data - */ -public class CloudStackTemplateAndIsoUsageData extends CloudStackUsageData { -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackUsageData.java index 01d4c3e..8b74020 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackUsageData.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackUsageData.java @@ -36,7 +36,7 @@ public String getTimeField() { @Override public TimeUnit getTimeUnit() { - return TimeUnit.SECONDS; + return TimeUnit.MILLISECONDS; } @Override @@ -53,7 +53,7 @@ public Map preProcess(Map original) { @Override public Boolean shouldPublish() { - return true; + return false; } @Override diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMAllocated.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMAllocated.java new file mode 100644 index 0000000..a327ec5 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMAllocated.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for VM Allocated + */ +public class CloudStackVMAllocated extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMRunning.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMRunning.java new file mode 100644 index 0000000..49b5864 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMRunning.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for VM Running + */ +public class CloudStackVMRunning extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMUsageData.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMUsageData.java deleted file mode 100644 index 3b33b93..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVMUsageData.java +++ /dev/null @@ -1,25 +0,0 @@ -package ch.icclab.cyclops.consume.data.model.cloudstack; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/** - * Author: Skoviera - * Created: 25/05/16 - * Description: POJO registration for CloudStack Virtual Machine Usage Data, both type Running and Allocated - */ -public class CloudStackVMUsageData extends CloudStackUsageData { -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVPNUser.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVPNUser.java new file mode 100644 index 0000000..0fa6699 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVPNUser.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for VPN User + */ +public class CloudStackVPNUser extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVolume.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVolume.java new file mode 100644 index 0000000..cb65ad6 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/cloudstack/CloudStackVolume.java @@ -0,0 +1,25 @@ +package ch.icclab.cyclops.consume.data.model.cloudstack; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Skoviera + * Created: 12/09/16 + * Description: Data mapping for Volume + */ +public class CloudStackVolume extends CloudStackUsageData { +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackIpTimeUDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackIpTimeUDR.java new file mode 100644 index 0000000..fe09ba0 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackIpTimeUDR.java @@ -0,0 +1,26 @@ +package ch.icclab.cyclops.consume.data.model.scaleup; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Oleksii + * Created: 01/06/16 + * Description: OpenStackEvent UDR measurement definition + */ +public class OpenStackIpTimeUDR extends OpenStackTimeUDR { + +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackEventUDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackTimeUDR.java similarity index 71% rename from core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackEventUDR.java rename to core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackTimeUDR.java index f76b360..a010cde 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/OpenStackEventUDR.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackTimeUDR.java @@ -1,4 +1,4 @@ -package ch.icclab.cyclops.consume.data.model; +package ch.icclab.cyclops.consume.data.model.scaleup; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -28,37 +28,26 @@ * Created: 01/06/16 * Description: OpenStackEvent UDR measurement definition */ -public class OpenStackEventUDR implements DataMapping{ +class OpenStackTimeUDR implements DataMapping { @Override - public String getTimeField() { - return "time"; - } + public String getTimeField() { return "time"; } @Override - public TimeUnit getTimeUnit() { - return TimeUnit.SECONDS; - } + public TimeUnit getTimeUnit() { return TimeUnit.SECONDS; } @Override public List getTagNames() { List list = new ArrayList<>(); - list.add("account"); list.add("instanceId"); return list; } @Override - public Map preProcess(Map original) { - return original; - } + public Map preProcess(Map original) { return original; } @Override - public Boolean shouldPublish() { - return true; - } + public Boolean shouldPublish() { return false; } @Override - public Boolean doNotBroadcastButRoute() { - return false; - } -} \ No newline at end of file + public Boolean doNotBroadcastButRoute() { return false; } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackUpTimeUDR.java b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackUpTimeUDR.java new file mode 100644 index 0000000..fa475cd --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/consume/data/model/scaleup/OpenStackUpTimeUDR.java @@ -0,0 +1,26 @@ +package ch.icclab.cyclops.consume.data.model.scaleup; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * Author: Oleksii + * Created: 01/06/16 + * Description: OpenStackEvent UDR measurement definition + */ +public class OpenStackUpTimeUDR extends OpenStackTimeUDR { + +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java new file mode 100644 index 0000000..c015654 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/AbstractEndpoint.java @@ -0,0 +1,28 @@ +package ch.icclab.cyclops.endpoint; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import org.restlet.resource.ServerResource; + +/** + * Author: Skoviera + * Created: 15/09/16 + * Description: Abstract Endpoint class + */ +public abstract class AbstractEndpoint extends ServerResource { + public abstract String getRoute(); +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java index a37e390..bdddaee 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/CommandEndpoint.java @@ -18,32 +18,23 @@ package ch.icclab.cyclops.endpoint; import ch.icclab.cyclops.consume.command.CommandConsumer; -import ch.icclab.cyclops.publish.Messenger; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import com.google.gson.Gson; import org.restlet.representation.Representation; import org.restlet.resource.Post; -import org.restlet.resource.ServerResource; import java.io.IOException; -import java.util.List; /** * Author: Skoviera * Created: 08/07/16 * Description: Handle uploading data frames (the same way as with RabbitMQ) */ -public class CommandEndpoint extends ServerResource { +public class CommandEndpoint extends AbstractEndpoint { - public static String ENDPOINT = "/command"; - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - // logger - final static Logger logger = LogManager.getLogger(CommandEndpoint.class.getName()); + @Override + public String getRoute() { + return "/command"; + } /** * Dispatch and process POST request based on provided parameter @@ -52,7 +43,6 @@ public class CommandEndpoint extends ServerResource { */ @Post public String processPost(Representation entity) throws IOException { - counter.increment(ENDPOINT); try { // first access data consumer @@ -61,20 +51,11 @@ public String processPost(Representation entity) throws IOException { // process the message consumer.consume(entity.getText()); - // get execution status + // get execution status and response CommandConsumer.ExecutionStatus status = consumer.getStatus(); + Object response = consumer.getResponse(); - // was successfully executed - if (status.wasExecuted()) { - - // request restful container - List list = Messenger.getInstance().retrieveRestfulContainer(); - - // return it if it's not empty - return (list != null && !list.isEmpty())? PrettyGson.toJson(list) : status.getMessage(); - } else { - return status.getMessage(); - } + return (status.wasExecuted() && response != null)? new Gson().toJson(response) : status.getMessage(); } catch (Exception e) { return String.format("Error: %s", e.getMessage()); diff --git a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java index 5d48d09..16a3c2a 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/DataEndpoint.java @@ -17,37 +17,50 @@ package ch.icclab.cyclops.endpoint; +import ch.icclab.cyclops.consume.command.model.generic.model.UDR; import ch.icclab.cyclops.consume.data.DataConsumer; +import ch.icclab.cyclops.consume.data.DataProcess; +import ch.icclab.cyclops.dto.Measurement; import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.Settings; +import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.load.model.PublisherCredentials; -import ch.icclab.cyclops.util.APICallCounter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import ch.icclab.cyclops.timeseries.InfluxDBClient; +import ch.icclab.cyclops.timeseries.InfluxDBResponse; +import ch.icclab.cyclops.timeseries.QueryBuilder; +import ch.icclab.cyclops.util.loggers.RESTLogger; +import com.google.gson.Gson; +import org.apache.commons.lang.math.NumberUtils; import org.restlet.representation.Representation; +import org.restlet.resource.Get; import org.restlet.resource.Post; -import org.restlet.resource.ServerResource; import java.io.IOException; +import java.util.List; +import java.util.Map; /** * Author: Skoviera * Created: 08/07/16 * Description: Handle uploading data frames (the same way as with RabbitMQ) */ -public class DataEndpoint extends ServerResource { +public class DataEndpoint extends AbstractEndpoint { - public static String ENDPOINT = "/data"; + private static String SELECTED_MEASUREMENT = UDR.class.getSimpleName(); // used for accessing DataConsumer private PublisherCredentials publisher; private String defaultName; - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); + // supported functions + private final static String FUN_COUNT = "count"; - // logger - final static Logger logger = LogManager.getLogger(DataEndpoint.class.getName()); + // supported params + private final static String PARAM_PAGE = "page"; + private final static String PARAM_FROM = "from"; + private final static String PARAM_TO = "to"; + + private final InfluxDBClient dbClient = new InfluxDBClient(); public DataEndpoint() { Settings settings = Loader.getSettings(); @@ -55,6 +68,144 @@ public DataEndpoint() { this.publisher = settings.getPublisherCredentials(); } + @Override + public String getRoute() { + return "/data"; + } + + @Get + public String processGet(){ + + // access query user specified + Map params = getQuery().getValuesMap(); + + // prepare measurement + Measurement measurement = new Measurement(SELECTED_MEASUREMENT, params); + + // has user provided page number? + Integer pageNumber = 0; + if (params.containsKey(PARAM_PAGE)) { + pageNumber = NumberUtils.toInt((String) params.get(PARAM_PAGE), 0); + params.remove(PARAM_PAGE); + } + + // prepare query + QueryBuilder builder = prepareQuery(measurement, params); + // get number of pages + Integer count = getCountForMeasurement(builder); + + // set page limit + Integer pageLimit = Loader.getSettings().getInfluxDBCredentials().getInfluxDBPageSizeLimit(); + measurement.setPageSize(pageLimit); + + // only if it makes sense + if (count > 0) { + // get data from InfluxDB + fillMeasurementWithRecordsFromDatabase(measurement, builder, pageNumber, pageLimit); + measurement.setTotalRecords(count); + } else { + measurement.setDisplayedRecords(0); + measurement.setTotalRecords(0); + } + + // don't forget to log it + RESTLogger.log(String.format("Serving %d records (out of %d) for measurement \"%s\" as page %d", measurement.getDisplayedRecords(), measurement.getTotalRecords(), measurement.getMeasurement(), measurement.getPageNumber())); + + // return measurement + return new Gson().toJson(measurement); + } + + /** + * Prepare query based on measurement and parameters + * @param measurement details + * @param params to be used + * @return QueryBuilder + */ + private QueryBuilder prepareQuery(Measurement measurement, Map params) { + // create query builder + QueryBuilder builder = new QueryBuilder(measurement.getMeasurement()); + + // date range specification FROM + if (params.containsKey(PARAM_FROM)) { + Long from = NumberUtils.toLong(params.get(PARAM_FROM), 0); + + // only if it makes sense + if (from > 0) { + builder.timeFrom(from, DataProcess.TIME_UNIT); + } + + params.remove(PARAM_FROM); + } + + // date range specification TO + if (params.containsKey(PARAM_TO)) { + Long to = NumberUtils.toLong(params.get(PARAM_TO), 0); + + // only if it makes sense + if (to > 0) { + builder.timeTo(to, DataProcess.TIME_UNIT); + } + + params.remove(PARAM_TO); + } + + // add where clauses iteratively + for (Map.Entry entry: params.entrySet()) { + String strNumber = entry.getValue(); + // if it is number + if (NumberUtils.isNumber(strNumber)) { + builder.where(entry.getKey(), NumberUtils.createDouble(strNumber)); + } else { + builder.where(entry.getKey(), entry.getValue()); + } + } + + return builder; + } + + /** + * Count number of records for specified query + * @param builder prepare query builder + * @return Integer + */ + private Integer getCountForMeasurement(QueryBuilder builder) { + try { + // add COUNT select and pass builder + InfluxDBResponse response = dbClient.executeQuery(builder.count(InfluxDBCredentials.COUNTER_FIELD_NAME)); + List countResult = response.getListOfObjects(); + + // get count from first record + return ((Double) countResult.get(0).get(FUN_COUNT)).intValue(); + } catch (Exception ignored) { + return 0; + } + } + + /** + * Add result of database query to measurement + * @param measurement to be filled + * @param builder prepared query + * @param pageNumber to be used + * @param pageLimit for pagination + */ + private void fillMeasurementWithRecordsFromDatabase(Measurement measurement, QueryBuilder builder, Integer pageNumber, Integer pageLimit) { + + try { + // reset selected fields first + builder.resetSelectClause(); + + // add limit and offset, plus execute query + InfluxDBResponse response = dbClient.executeQuery(builder.limit(pageLimit).offset(pageNumber*pageLimit)); + List parsed = response.getListOfObjects(); + + // update measurement + measurement.setData(parsed); + measurement.setPageNumber(pageNumber); + measurement.setDisplayedRecords(parsed.size()); + } catch (Exception ignored) {} + + } + /** * Dispatch and process POST request based on provided parameter * @param entity json @@ -62,7 +213,6 @@ public DataEndpoint() { */ @Post public String processPost(Representation entity) throws IOException { - counter.increment(ENDPOINT); try { // first access data consumer @@ -71,11 +221,7 @@ public String processPost(Representation entity) throws IOException { // process the message consumer.consume(entity.getText()); - // get stats - Long valid = consumer.getNumberOfValidRecords(); - - - return (valid > 0) ? String.format("%d processed", valid) : "Invalid JSON"; + return "Added for processing"; } catch (Exception e) { return String.format("Error: %s", e.getMessage()); diff --git a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java deleted file mode 100644 index 84e6142..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/ListEndpoint.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.endpoint; - -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: List available endpoints - */ -public class ListEndpoint extends ServerResource { - - public static String ENDPOINT = "/list"; - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - /** - * This method will return JSON stats for APICallCounter object - * @return JSON - */ - @Get - public String processCommand() { - return PrettyGson.toJson(counter.getAvailableEndpoints()); - } -} \ No newline at end of file diff --git a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java deleted file mode 100644 index 7266745..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementEndpoint.java +++ /dev/null @@ -1,194 +0,0 @@ -package ch.icclab.cyclops.endpoint; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.consume.data.DataConsumer; -import ch.icclab.cyclops.dto.Measurement; -import ch.icclab.cyclops.load.Loader; -import ch.icclab.cyclops.load.model.InfluxDBCredentials; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.QueryBuilder; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import ch.icclab.cyclops.util.loggers.RESTLogger; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Endpoint for measurements - */ -public class MeasurementEndpoint extends ServerResource{ - final static Logger logger = LogManager.getLogger(MeasurementEndpoint.class.getName()); - - public static String ENDPOINT = "/measurement"; - public static String ATTRIBUTE = "name"; - - // supported functions - private final static String FUN_COUNT = "count"; - - // supported params - private final static String PARAM_PAGE = "page"; - private final static String PARAM_FROM = "from"; - private final static String PARAM_TO = "to"; - - private final APICallCounter counter = APICallCounter.getInstance(); - private final InfluxDBClient dbClient = InfluxDBClient.getInstance(); - private String attribute; - - /** - * Copy parameters for this particular request - */ - public void doInit() { - attribute = (String) getRequestAttributes().get(ATTRIBUTE); - } - - @Get - public String processGet(){ - counter.increment(ENDPOINT); - - // access query user specified - Map params = getQuery().getValuesMap(); - - // prepare measurement - Measurement measurement = new Measurement(attribute, params); - - // has user provided page number? - Integer pageNumber = 0; - if (params.containsKey(PARAM_PAGE)) { - pageNumber = NumberUtils.toInt((String) params.get(PARAM_PAGE), 0); - params.remove(PARAM_PAGE); - } - - // prepare query - QueryBuilder builder = prepareQuery(measurement, params); - // get number of pages - Integer count = getCountForMeasurement(builder); - - // set page limit - Integer pageLimit = Loader.getSettings().getInfluxDBCredentials().getInfluxDBPageSizeLimit(); - measurement.setPageSize(pageLimit); - - // only if it makes sense - if (count > 0) { - // get data from InfluxDB - fillMeasurementWithRecordsFromDatabase(measurement, builder, pageNumber, pageLimit); - measurement.setTotalRecords(count); - } else { - measurement.setDisplayedRecords(0); - measurement.setTotalRecords(0); - } - - // don't forget to log it - RESTLogger.log(String.format("Serving %d records (out of %d) for measurement \"%s\" as page %d", measurement.getDisplayedRecords(), measurement.getTotalRecords(), measurement.getMeasurement(), measurement.getPageNumber())); - - // return measurement - return PrettyGson.toJson(measurement); - } - - /** - * Prepare query based on measurement and parameters - * @param measurement details - * @param params to be used - * @return QueryBuilder - */ - private QueryBuilder prepareQuery(Measurement measurement, Map params) { - // create query builder - QueryBuilder builder = new QueryBuilder(measurement.getMeasurement()); - - // date range specification FROM - if (params.containsKey(PARAM_FROM)) { - Long from = NumberUtils.toLong(params.get(PARAM_FROM), 0); - - // only if it makes sense - if (from > 0) { - builder.timeFrom(from, DataConsumer.TIME_UNIT); - } - - params.remove(PARAM_FROM); - } - - // date range specification TO - if (params.containsKey(PARAM_TO)) { - Long to = NumberUtils.toLong(params.get(PARAM_TO), 0); - - // only if it makes sense - if (to > 0) { - builder.timeTo(to, DataConsumer.TIME_UNIT); - } - - params.remove(PARAM_TO); - } - - // add where clauses iteratively - for (Map.Entry entry: params.entrySet()) { - String strNumber = entry.getValue(); - // if it is number - if (NumberUtils.isNumber(strNumber)) { - builder.where(entry.getKey(), NumberUtils.createDouble(strNumber)); - } else { - builder.where(entry.getKey(), entry.getValue()); - } - } - - return builder; - } - - /** - * Count number of records for specified query - * @param builder prepare query builder - * @return Integer - */ - private Integer getCountForMeasurement(QueryBuilder builder) { - try { - // add COUNT select and pass builder - List countResult = dbClient.executeQuery(builder.count(InfluxDBCredentials.COUNTER_FIELD_NAME)); - - // get count from first record - return ((Double) countResult.get(0).get(FUN_COUNT)).intValue(); - } catch (Exception ignored) { - return 0; - } - } - - /** - * Add result of database query to measurement - * @param measurement to be filled - * @param builder prepared query - * @param pageNumber to be used - * @param pageLimit for pagination - */ - private void fillMeasurementWithRecordsFromDatabase(Measurement measurement, QueryBuilder builder, Integer pageNumber, Integer pageLimit) { - - // reset selected fields first - builder.resetSelectClause(); - - // add limit and offset, plus execute query - List parsed = dbClient.executeQuery(builder.limit(pageLimit).offset(pageNumber*pageLimit)); - - // update measurement - measurement.setData(parsed); - measurement.setPageNumber(pageNumber); - measurement.setDisplayedRecords(parsed.size()); - } -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java deleted file mode 100644 index 9ab58c7..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/MeasurementsEndpoint.java +++ /dev/null @@ -1,73 +0,0 @@ -package ch.icclab.cyclops.endpoint; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.load.Loader; -import ch.icclab.cyclops.timeseries.InfluxDBClient; -import ch.icclab.cyclops.timeseries.QueryBuilder; -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Endpoint for measurements - */ -public class MeasurementsEndpoint extends ServerResource{ - final static Logger logger = LogManager.getLogger(MeasurementsEndpoint.class.getName()); - - public static String ENDPOINT = "/measurements"; - - private final APICallCounter counter = APICallCounter.getInstance(); - private final InfluxDBClient dbClient = InfluxDBClient.getInstance(); - - @Get - public String processGet(){ - counter.increment(ENDPOINT); - - // we will list measurements - QueryBuilder builder = QueryBuilder.getMeasurementsQuery(); - - // get list of measurements - List result = dbClient.executeQuery(builder); - - List list = new ArrayList<>(); - - // iterate over list of maps and add to the list only names - for (Map map: result) { - if (map.containsKey(InfluxDBClient.MEASUREMENT_FIELD_NAME)) { - String name = (String) map.get(InfluxDBClient.MEASUREMENT_FIELD_NAME); - - // we are returning only measurement names that are not the default container - if (!name.equals(Loader.getSettings().getInfluxDBCredentials().getInfluxDBDefaultMeasurement())) { - list.add(name); - } - } - } - - // return as JSON - return PrettyGson.toJson(list); - } -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java index a0c8e74..0dc32a4 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java @@ -17,25 +17,22 @@ package ch.icclab.cyclops.endpoint; -import ch.icclab.cyclops.util.APICallCounter; import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; /** * Author: Skoviera * Created: 21/01/16 * Description: Serve application's version over root endpoint */ -public class RootEndpoint extends ServerResource { - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - public static String ENDPOINT = "/"; +public class RootEndpoint extends AbstractEndpoint { @Get public String root(){ - counter.increment(ENDPOINT); - return "RCB UDR micro service - version 1.0.1"; + return "RCB BOX micro service - version 2.1.0"; + } + + @Override + public String getRoute() { + return "/"; } } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java b/core/udr/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java deleted file mode 100644 index ca0cada..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/endpoint/StatusEndpoint.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.endpoint; - -import ch.icclab.cyclops.util.APICallCounter; -import ch.icclab.cyclops.util.PrettyGson; -import org.restlet.resource.Get; -import org.restlet.resource.ServerResource; -import java.util.HashMap; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: This class handles the internal APICallCounter - */ -public class StatusEndpoint extends ServerResource { - - // used as counter - private APICallCounter counter = APICallCounter.getInstance(); - - public static String ENDPOINT = "/status"; - - /** - * This method will return JSON stats for APICallCounter object - * @return JSON - */ - @Get - public String processCommand() { - // first get running statistics - HashMap stats = counter.getRunningStats(); - - // and return - return PrettyGson.toJson(stats); - } -} \ No newline at end of file diff --git a/core/udr/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java b/core/udr/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java new file mode 100644 index 0000000..fb01a40 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/executor/TaskExecutor.java @@ -0,0 +1,61 @@ +package ch.icclab.cyclops.executor; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Global task executor + */ +public class TaskExecutor { + final static Logger logger = LogManager.getLogger(TaskExecutor.class.getName()); + private static TaskExecutor singleton = new TaskExecutor(); + + private ExecutorService executor; + + private TaskExecutor(){ + executor = obtainSession(); + } + + private ExecutorService obtainSession() { + // create a pool threads (based on available cpu cores) + return Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + } + + public static TaskExecutor getInstance() { return singleton; } + + public void addTask(Runnable task) { + if (executor == null) { + executor = obtainSession(); + } + executor.submit(task); + } + + public void shutDown() { + if (executor != null) { + logger.trace("Shutting down Task Executor"); + executor.shutdownNow(); + executor = null; + } + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/publish/APICaller.java b/core/udr/src/main/java/ch/icclab/cyclops/publish/APICaller.java new file mode 100644 index 0000000..2412449 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/publish/APICaller.java @@ -0,0 +1,117 @@ +package ch.icclab.cyclops.publish; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.util.BeanList; +import com.google.gson.Gson; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.net.URL; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 29/07/16 + * Description: Call API endpoint + */ +public class APICaller { + + public class Response { + private String object; + private int status; + + public Response(String obj, int stat) { + object = obj; + status = stat; + } + + public int getStatus() { + return status; + } + + public String getAsString() throws Exception { + return object; + } + + public List getAsList() throws Exception { + return new Gson().fromJson(object, List.class); + } + + public Map getAsMap() throws Exception { + return new Gson().fromJson(object, Map.class); + } + + public Object getAsClass(Class clazz) throws Exception { + return new Gson().fromJson(object, clazz); + } + + public List getAsListOfType(Class clazz) throws Exception { + List list = new Gson().fromJson(object, List.class); + return BeanList.populate(list, clazz); + } + } + + /** + * Perform POST query and return Response + * @param endpoint to be called + * @param object to be passed + * @return Response object + * @throws Exception + */ + public APICaller.Response post(URL endpoint, Object object) throws Exception { + // prepare connection + HttpClient client = HttpClientBuilder.create().build(); + + // create request + HttpPost request = new HttpPost(endpoint.toURI()); + StringEntity entity = new StringEntity(new Gson().toJson(object)); + request.addHeader("Accept", "application/json"); + request.addHeader("Content-Type", "application/json"); + request.setEntity(entity); + + // execute response + HttpResponse response = client.execute(request); + return new Response(IOUtils.toString(response.getEntity().getContent()), response.getStatusLine().getStatusCode()); + } + + /** + * Perform GET query and return Response + * @param endpoint to be called + * @return Response object + * @throws Exception + */ + public APICaller.Response get(URL endpoint) throws Exception { + // prepare connection + HttpClient client = HttpClientBuilder.create().build(); + + // create request + HttpGet request = new HttpGet(endpoint.toURI()); + request.addHeader("Accept", "application/json"); + request.addHeader("Content-Type", "application/json"); + + // execute response + HttpResponse response = client.execute(request); + return new Response(IOUtils.toString(response.getEntity().getContent()), response.getStatusLine().getStatusCode()); + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/publish/Messenger.java b/core/udr/src/main/java/ch/icclab/cyclops/publish/Messenger.java index 5d7e9dd..796d5e7 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/publish/Messenger.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/publish/Messenger.java @@ -19,6 +19,7 @@ import ch.icclab.cyclops.util.loggers.DispatchLogger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.List; diff --git a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java index a582d4e..b1850ee 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/BatchPointsContainer.java @@ -29,6 +29,7 @@ * Description: Batch session for InfluxDB */ public class BatchPointsContainer { + // if you ever run addPoint in parallel stream, make sure points are encapsulated with Collections.synchronizedList() private List points = new ArrayList<>(); public void addPoint(Point.Builder builder) { @@ -37,7 +38,7 @@ public void addPoint(Point.Builder builder) { public BatchPoints getPoints() { // get empty container - BatchPoints container = InfluxDBClient.getInstance().getEmptyContainer(); + BatchPoints container = new InfluxDBClient().getEmptyContainer(); // iterate over points for (Point.Builder builder : points) { @@ -47,4 +48,12 @@ public BatchPoints getPoints() { return container; } + + public Integer size() { + return points.size(); + } + + public Point.Builder getFirstPoint() { + return (size() > 0) ? points.get(0): null; + } } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java new file mode 100644 index 0000000..fb87c61 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/GenerateDBPoint.java @@ -0,0 +1,118 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import com.github.wnameless.json.flattener.JsonFlattener; +import com.google.gson.Gson; +import org.influxdb.dto.Point; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Author: Skoviera + * Created: 02/09/16 + * Description: Helper class for DB point generation + */ +public class GenerateDBPoint { + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @return Point.Builder + */ + public static Point.Builder fromObject(Object object) { + return generate(object, null, null, null); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param timeField name + * @param unit of time + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTime(Object object, String timeField, TimeUnit unit) { + return generate(object, timeField, unit, null); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param tags list + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTags(Object object, List tags) { + return generate(object, null, null, tags); + } + + /** + * Generate DB Point Builder from specified object + * @param object to be processed + * @param timeField name + * @param unit of time + * @param tags list + * @return Point.Builder + */ + public static Point.Builder fromObjectWithTimeAndTags(Object object, String timeField, TimeUnit unit, List tags) { + return generate(object, timeField, unit, tags); + } + + /** + * Generate a DBPoint + * @param object to be serialised + * @param time field + * @param unit of time + * @param tags list of tags + * @return Point.Builder or null + */ + private static Point.Builder generate(Object object, String time, TimeUnit unit, List tags) { + try { + // flatten possibly nested object + Map flat = JsonFlattener.flattenAsMap(new Gson().toJson(object)); + + // create point builder + Point.Builder builder = Point.measurement(object.getClass().getSimpleName()); + + // if time field was provided + if (time != null && unit != null && flat.containsKey(time)) { + builder.time(TimeStamp.cast(flat.get(time)), unit); + flat.remove(time); + } else { + builder.time(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + } + + // if list of tags was provided + if (tags != null && !tags.isEmpty()) { + for (String tag: tags) { + if (flat.containsKey(tag) && flat.get(tag) instanceof String) { + builder.tag(tag, (String) flat.get(tag)); + flat.remove(tag); + } + } + } + + // finally add individual fields + flat.entrySet().stream().filter(entry -> entry.getValue() != null).forEach(entry -> builder.field(entry.getKey(), entry.getValue())); + + return builder; + } catch (Exception ignored) { + return null; + } + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java index 688a6dd..f509425 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBClient.java @@ -17,10 +17,13 @@ package ch.icclab.cyclops.timeseries; +import ch.icclab.cyclops.load.Loader; import ch.icclab.cyclops.load.model.InfluxDBCredentials; import ch.icclab.cyclops.util.loggers.TimeSeriesLogger; +import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.atteo.evo.inflector.English; import org.influxdb.InfluxDB; import org.influxdb.InfluxDBFactory; import org.influxdb.dto.BatchPoints; @@ -28,8 +31,10 @@ import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; +import java.util.Collections; import java.util.List; -import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Author: Skoviera @@ -38,38 +43,23 @@ */ public class InfluxDBClient { final static Logger logger = LogManager.getLogger(InfluxDBClient.class.getName()); - public static String MEASUREMENT_FIELD_NAME = "name"; // singleton - private static InfluxDBClient singleton; private InfluxDBCredentials credentials; - - /** - * Create InfluxDB instance - * @param conf to be used - */ - public static void createInstance(InfluxDBCredentials conf){ - if (singleton == null) { - singleton = new InfluxDBClient(conf); - } - } + private InfluxDB session; /** * Constructor * @param conf to be used */ - private InfluxDBClient(InfluxDBCredentials conf) { + public InfluxDBClient(InfluxDBCredentials conf) { credentials = conf; - - createDatabases(credentials.getInfluxDBTSDB()); + session = obtainSession(); } - /** - * Simple implementation of Singleton class - * @return instance of InfluxDB object - */ - public static InfluxDBClient getInstance() { - return singleton; + public InfluxDBClient() { + credentials = Loader.getSettings().getInfluxDBCredentials(); + session = obtainSession(); } /** @@ -80,6 +70,24 @@ private InfluxDB obtainSession() { return InfluxDBFactory.connect(credentials.getInfluxDBURL(), credentials.getInfluxDBUsername(), credentials.getInfluxDBPassword()); } + /** + * Ping InfluxDB server to see whether it is alive + * @throws Exception + */ + public void ping() throws Exception { + session.ping(); + } + + /** + * Enable batch processing for items that are added as single points + * @param flushPoints flush every X points (for example 2000) + * @param flushFrequency flush every Y time unit (for example 100 ms) + * @param unit time unit (for example milliseconds) + */ + public void configureBatchOnSinglePoints(Integer flushPoints, Integer flushFrequency, TimeUnit unit) { + session.enableBatch(flushPoints, flushFrequency, unit); + } + /** * Save container to InfluxDB * @param container to be persisted @@ -88,9 +96,8 @@ public void persistContainer(BatchPointsContainer container) { persistContainer(container.getPoints()); } private void persistContainer(BatchPoints container) { - InfluxDB con = obtainSession(); TimeSeriesLogger.log(String.format("Saving container with %d points to database", container.getPoints().size())); - con.write(container); + session.write(container); } /** @@ -106,14 +113,11 @@ protected BatchPoints getEmptyContainer() { * @param builder to generate point */ public void persistSinglePoint(Point.Builder builder) { - BatchPoints container = getEmptyContainer(); - // add mandatory hidden field Point point = builder.addField(InfluxDBCredentials.COUNTER_FIELD_NAME, true).build(); - container.point(point); - - persistContainer(container); + // depending on whether batch processing for single points is enabled store immediately or wait for flush + session.write(credentials.getInfluxDBTSDB(), "default", point); } /** @@ -121,41 +125,38 @@ public void persistSinglePoint(Point.Builder builder) { * @param names for database creation */ public void createDatabases(String ... names) { - InfluxDB client = obtainSession(); - // now create required databases for (String name: names) { TimeSeriesLogger.log(String.format("Making sure \"%s\" database exists", name)); - client.createDatabase(name); + session.createDatabase(name); } } /** * Execute query * @param builder QueryBuilder - * @return QueryResult + * @return InfluxDBResponse or null */ - public List executeQuery(QueryBuilder builder) { - InfluxDB client = obtainSession(); + public InfluxDBResponse executeQuery(QueryBuilder builder) { + return executeQuery(Collections.singletonList(builder)); + } + public InfluxDBResponse executeQuery(List builders) { + try { + TimeSeriesLogger.log(String.format("About to execute %d %s", builders.size(), English.plural("query", builders.size()))); - // execute query - QueryResult result = client.query(new Query(builder.build(), credentials.getInfluxDBTSDB())); + // concatenate multiple queries into one + List queries = builders.stream().map(QueryBuilder::build).collect(Collectors.toList()); + String multipleQuery = StringUtils.join(queries, ";"); - // return it parsed as list of maps - return ParseQueryResult.parse(result); - } + // connect to InfluxDB and execute query + QueryResult result = session.query(new Query(multipleQuery, credentials.getInfluxDBTSDB())); - /** - * Execute query and map it to specified class - * @param builder query - * @param clazz for mapping - * @return list or null - */ - public List executeQueryAndMapItToClass(QueryBuilder builder, Class clazz) { - // first execute query and get results - List result = executeQuery(builder); + // return InfluxDB response or null + return (!result.hasError())? new InfluxDBResponse(result): null; - // now return it parsed - return ParseQueryResult.map(result, clazz); + } catch (Exception ignored) { + TimeSeriesLogger.log(String.format("Query execution failed: %s", ignored.getMessage())); + return null; + } } } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java new file mode 100644 index 0000000..f94abac --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/InfluxDBResponse.java @@ -0,0 +1,136 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import ch.icclab.cyclops.load.model.InfluxDBCredentials; +import ch.icclab.cyclops.util.BeanList; +import com.github.wnameless.json.unflattener.JsonUnflattener; +import com.google.gson.Gson; +import org.influxdb.dto.QueryResult; +import org.joda.time.DateTime; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: InfluxDB response container + */ +public class InfluxDBResponse { + + private static String TIME_FIELD = "time"; + + private QueryResult object; + + public InfluxDBResponse(QueryResult object) { + this.object = object; + } + + /** + * Output underlying InfluxDB structure in JSON + * @return flat table structure + */ + public String getTableAsJson() throws Exception { + return new Gson().toJson(this.object); + + } + + /** + * Parse and transform InfluxDB's underlying structure + * @return JSON + */ + public String getListOfObjectsAsJson() throws Exception { + return new Gson().toJson(getListOfObjects()); + } + + /** + * Parse and transform InfluxDB's underlying structure + * @param clazz definition + * @return List + */ + public List getAsListOfType(Class clazz) throws Exception { + List list = getListOfObjects(); + return BeanList.populate(list, clazz); + } + + /** + * Parse and transform InfluxDB's underlying structure + * @return list of maps + */ + public List getListOfObjects() throws Exception { + + List list = new ArrayList<>(); + + if (object != null && !object.hasError()) { + // iterate over available results + for (QueryResult.Result result: object.getResults()) { + // multiple query responses can have empty results + if (result != null) { + List bulk = result.getSeries(); + if (bulk != null && !bulk.isEmpty()) { + // iterate over available series + for (QueryResult.Series series: result.getSeries()) { + // iterate over individual value sets + Map tags = series.getTags(); + List columns = series.getColumns(); + int time = columns.indexOf(TIME_FIELD); + List> values = series.getValues(); + + // iterate over individual values + if (values != null && !values.isEmpty()) { + for (List value: values) { + // we will store one row here + Map row = new HashMap<>(); + + // add tag values if there are any + if (tags != null && !tags.isEmpty()) { + row.putAll(tags); + } + + // individual entries of the value + int i = 0; + for (Object entry: value) { + // timestamp needs to be in Long and not String + if (time == i) { + entry = new DateTime(entry).getMillis() / 1000; + } + + row.put(columns.get(i), entry); + i++; + } + + // remove property that is used for counting and should never be visible + row.remove(InfluxDBCredentials.COUNTER_FIELD_NAME); + + // return de-flattened map + list.add(new Gson().fromJson(JsonUnflattener.unflatten(new Gson().toJson(row)), Map.class)); + } + } + } + } + } + } + + return list; + } + + throw new Exception("Couldn't parse InfluxDB response"); + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java deleted file mode 100644 index 4ba0769..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/ParseQueryResult.java +++ /dev/null @@ -1,128 +0,0 @@ -package ch.icclab.cyclops.timeseries; -/* - * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import ch.icclab.cyclops.load.model.InfluxDBCredentials; -import com.github.wnameless.json.unflattener.JsonUnflattener; -import com.google.gson.Gson; -import org.apache.commons.beanutils.BeanUtils; -import org.influxdb.dto.QueryResult; -import org.joda.time.DateTime; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Author: Skoviera - * Created: 17/05/16 - * Description: Parse InfluxDB's Query Result - */ -public class ParseQueryResult { - - private static String TIME_FIELD = "time"; - - public static List parse(QueryResult data) { - // container for result - List container = new ArrayList<>(); - - // mapping - Gson gson = new Gson(); - - try { - // go over all results - for (QueryResult.Result result : data.getResults()) { - - // go over individual series - for (QueryResult.Series serie : result.getSeries()) { - - // store columns here - List columns = new ArrayList<>(); - columns.addAll(serie.getColumns()); - - // get time field position - int time = columns.indexOf(TIME_FIELD); - - // iterate over values - for (List values : serie.getValues()) { - // we will store one row here - Map row = new HashMap<>(); - - // add tag values if there are any - Map tags = serie.getTags(); - if (tags != null && !tags.isEmpty()) { - row.putAll(tags); - } - - // access objects - int i = 0; - for (Object value: values) { - - // timestamp needs to be in Long and not String - if (time == i) { - try { - // return seconds not milliseconds - value = new DateTime(value).getMillis() / 1000; - } catch (Exception ignored) { - } - } - - row.put(columns.get(i), value); - i++; - } - - // delete meta field - row.remove(InfluxDBCredentials.COUNTER_FIELD_NAME); - - // de-flatten structure - String rich = JsonUnflattener.unflatten(gson.toJson(row)); - container.add(gson.fromJson(rich, Map.class)); - } - } - } - } catch (Exception ignored) { - // in case of empty QueryResult body do nothing - } - - return container; - } - - public static List map(List list, Class clazz) { - List mapped = new ArrayList<>(); - - // iterate and map those objects - if (list != null) { - for (Map map : list) { - try { - Object bean = clazz.newInstance(); - - // map HashMap to POJO - BeanUtils.populate(bean, map); - - // add it to list of mapped CDRs - mapped.add(bean); - - } catch (Exception ignored) { - return null; - } - } - } - - return mapped; - } -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/QueryBuilder.java b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/QueryBuilder.java index 781214d..be6cbab 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/QueryBuilder.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/QueryBuilder.java @@ -361,4 +361,4 @@ private String getTimeDurationLetter(TimeUnit unit) { return "s"; } } -} \ No newline at end of file +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/util/PrettyGson.java b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java similarity index 61% rename from core/udr/src/main/java/ch/icclab/cyclops/util/PrettyGson.java rename to core/udr/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java index d54d16a..f64855f 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/util/PrettyGson.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/SharedInfluxDBSession.java @@ -1,4 +1,4 @@ -package ch.icclab.cyclops.util; +package ch.icclab.cyclops.timeseries; /* * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften * All Rights Reserved. @@ -16,21 +16,22 @@ * under the License. */ -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import java.util.concurrent.TimeUnit; /** * Author: Skoviera - * Created: 08/03/16 - * Description: Always use pretty JSON output + * Created: 31/08/16 + * Description: Shared InfluxDB session */ -public class PrettyGson { +public class SharedInfluxDBSession { + private static InfluxDBClient session; - private static Gson getGson() { - return new GsonBuilder().setPrettyPrinting().create(); - } + public static InfluxDBClient getSession() { + if (session == null) { + session = new InfluxDBClient(); + session.configureBatchOnSinglePoints(2000, 100, TimeUnit.MILLISECONDS); + } - public static String toJson(Object object) { - return getGson().toJson(object); + return session; } } diff --git a/core/udr/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java new file mode 100644 index 0000000..f77a4db --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/timeseries/TimeStamp.java @@ -0,0 +1,53 @@ +package ch.icclab.cyclops.timeseries; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import java.math.BigDecimal; + +/** + * Author: Skoviera + * Created: 06/09/16 + * Description: Various utils for TimeStamp object manipulation + */ +public class TimeStamp { + + /** + * Parse long from unknown type + * @param unknown object + * @return Long or null + */ + public static Long cast(Object unknown) { + try { + BigDecimal dec = (BigDecimal) unknown; + return dec.longValue(); + } catch (Exception a) { + try { + return Long.parseLong((String) unknown); + } catch (Exception b) { + try { + return (Long) unknown; + } catch (Exception c) { + try { + return Long.valueOf(unknown.toString()); + } catch (Exception d) { + return null; + } + } + } + } + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/util/APICallCounter.java b/core/udr/src/main/java/ch/icclab/cyclops/util/APICallCounter.java deleted file mode 100644 index ff640c1..0000000 --- a/core/udr/src/main/java/ch/icclab/cyclops/util/APICallCounter.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package ch.icclab.cyclops.util; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** - * Author: Skoviera - * Created: 21/01/16 - * Description: Class that will count API Calls and then provide the information (and reset counters) - */ -public class APICallCounter { - // we need a singleton for this - private static APICallCounter singleton = new APICallCounter(); - - // Hash-map to be used for counting - private HashMap map; - - // list used for holding them all - private ArrayList list; - - // Hash-map used as a template (for registering fields) - private HashMap template; - - /** - * Private Constructor is necessary for the singleton model - */ - private APICallCounter() { - template = new HashMap<>(); - list = new ArrayList<>(); - map = new HashMap<>(); - } - - /** - * Simply return singleton instance - * @return APICallCounter instance - */ - public static APICallCounter getInstance() { - return singleton; - } - - /** - * Will increment the counter for provided key - * In case that this key was not previously registered, it will add it to the hash-map - * However on dump, it will get removed and only the registered ones will stay - * @param key meaning API endpoint - */ - public void increment(String key) { - Object value = map.get(key); - - // now save new value or increment it - if (value == null) { - map.put(key, 1); - } else { - map.put(key, ((Integer) value) + 1); - } - } - - /** - * Register an API endpoint to the counter - * @param key as endpoint string - */ - public void registerEndpoint(String key) { - template.put(key, 0); - map.put(key, 0); - list.add(key); - } - - /** - * Register an API endpoint for listing - * @param key as endpoint - */ - public void registerEndpointWithoutCounting(String key) { - list.add(key); - } - - /** - * Will ask for running stats and DUMP the old hash-map so it can start from zero again - * @return hash-map - */ - public HashMap getRunningStats() { - // make a deep copy - HashMap stats = new HashMap(); - stats.putAll(map); - - // reset map based on registered entries in template - map = new HashMap(); - map.putAll(template); - - // return deep copy - return stats; - } - - /** - * Return list of available endpoints - * @return list - */ - public List getAvailableEndpoints() { - return list; - } -} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/util/BeanList.java b/core/udr/src/main/java/ch/icclab/cyclops/util/BeanList.java new file mode 100644 index 0000000..6c9d265 --- /dev/null +++ b/core/udr/src/main/java/ch/icclab/cyclops/util/BeanList.java @@ -0,0 +1,54 @@ +package ch.icclab.cyclops.util; +/* + * Copyright (c) 2016. Zuercher Hochschule fuer Angewandte Wissenschaften + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.beanutils.BeanUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Author: Skoviera + * Created: 09/08/16 + * Description: Utility to populate list of beans + */ +public class BeanList { + public static List populate(List list, Class clazz) { + List mapped = new ArrayList<>(); + + // iterate and map those objects + if (list != null) { + for (Map map : list) { + try { + Object bean = clazz.newInstance(); + + // map HashMap to POJO + BeanUtils.populate(bean, map); + + // add it to list of mapped CDRs + mapped.add(bean); + + } catch (Exception ignored) { + return null; + } + } + } + + return mapped; + } +} diff --git a/core/udr/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java b/core/udr/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java index de24ebf..71177d9 100644 --- a/core/udr/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java +++ b/core/udr/src/main/java/ch/icclab/cyclops/util/ShutDownListener.java @@ -17,6 +17,7 @@ */ import ch.icclab.cyclops.consume.RabbitMQListener; +import ch.icclab.cyclops.executor.TaskExecutor; import ch.icclab.cyclops.publish.RabbitMQPublisher; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -34,6 +35,9 @@ public class ShutDownListener extends Thread{ public void run() { logger.trace("We are shutting down UDR micro service"); + // shut down executor service + TaskExecutor.getInstance().shutDown(); + // and Consumer (RabbitMQ) RabbitMQListener.shutDown(); diff --git a/core/udr/src/main/resources/log4j2.component.properties b/core/udr/src/main/resources/log4j2.component.properties new file mode 100644 index 0000000..683a3da --- /dev/null +++ b/core/udr/src/main/resources/log4j2.component.properties @@ -0,0 +1 @@ +Log4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector \ No newline at end of file diff --git a/core/udr/src/main/resources/log4j2.xml b/core/udr/src/main/resources/log4j2.xml index d7995f7..7ca9460 100644 --- a/core/udr/src/main/resources/log4j2.xml +++ b/core/udr/src/main/resources/log4j2.xml @@ -1,27 +1,27 @@ - - - - - - - + + + + + + + - - + + - - + + - - + + - - + + - +