-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy path15282771049157.html
915 lines (626 loc) · 50.3 KB
/
15282771049157.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
<!doctype html>
<html class="no-js" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>
ELK: Using a Centralized Logging Architecture - Junkman
</title>
<link href="atom.xml" rel="alternate" title="Junkman" type="application/atom+xml">
<link rel="stylesheet" href="asset/css/foundation.min.css" />
<link rel="stylesheet" href="asset/css/docs.css" />
<script src="asset/js/vendor/modernizr.js"></script>
<script src="asset/js/vendor/jquery.js"></script>
<script src="asset/highlightjs/highlight.pack.js"></script>
<link href="asset/highlightjs/styles/github.css" media="screen, projection" rel="stylesheet" type="text/css">
<script>hljs.initHighlightingOnLoad();</script>
<script type="text/javascript">
function before_search(){
var searchVal = 'site:panlw.github.io ' + document.getElementById('search_input').value;
document.getElementById('search_q').value = searchVal;
return true;
}
</script>
</head>
<body class="antialiased hide-extras">
<div class="marketing off-canvas-wrap" data-offcanvas>
<div class="inner-wrap">
<nav class="top-bar docs-bar hide-for-small" data-topbar>
<section class="top-bar-section">
<div class="row">
<div style="position: relative;width:100%;"><div style="position: absolute; width:100%;">
<ul id="main-menu" class="left">
<li id=""><a target="self" href="index.html">Home</a></li>
<li id=""><a target="_self" href="archives.html">Archives</a></li>
</ul>
<ul class="right" id="search-wrap">
<li>
<form target="_blank" onsubmit="return before_search();" action="http://google.com/search" method="get">
<input type="hidden" id="search_q" name="q" value="" />
<input tabindex="1" type="search" id="search_input" placeholder="Search"/>
</form>
</li>
</ul>
</div></div>
</div>
</section>
</nav>
<nav class="tab-bar show-for-small">
<a href="javascript:void(0)" class="left-off-canvas-toggle menu-icon">
<span> Junkman</span>
</a>
</nav>
<aside class="left-off-canvas-menu">
<ul class="off-canvas-list">
<li><a href="index.html">HOME</a></li>
<li><a href="archives.html">Archives</a></li>
<li><a href="about.html">ABOUT</a></li>
<li><label>Categories</label></li>
<li><a href="Infra.html">Infra</a></li>
<li><a href="Coding.html">Coding</a></li>
<li><a href="Modeling.html">Modeling</a></li>
<li><a href="Archtecting.html">Archtecting</a></li>
</ul>
</aside>
<a class="exit-off-canvas" href="#"></a>
<section id="main-content" role="main" class="scroll-container">
<script type="text/javascript">
$(function(){
$('#menu_item_index').addClass('is_active');
});
</script>
<div class="row">
<div class="large-8 medium-8 columns">
<div class="markdown-body article-wrap">
<div class="article">
<h1>ELK: Using a Centralized Logging Architecture</h1>
<div class="read-more clearfix">
<span class="date">2018/6/6</span>
<span>posted in </span>
<span class="posted-in"><a href='Logging.html'>Logging</a></span>
<span class="comments">
</span>
</div>
</div><!-- article -->
<div class="article-content">
<blockquote>
<p>by Alexandre Lourenco · Mar. 07, 15 · DevOps Zone · Tutorial<br/>
<a href="https://dzone.com/articles/elk-using-centralized-logging">https://dzone.com/articles/elk-using-centralized-logging</a><br/>
<a href="https://dzone.com/articles/elk-using-centralized-logging-0">https://dzone.com/articles/elk-using-centralized-logging-0</a></p>
</blockquote>
<p>Welcome, dear reader, to another post from my blog. On this new series, we will talk about a architecture specially designed to process data from log files coming from applications, with the junction of 3 tools, Logstash, ElasticSearch and Kibana. But after all, do we really need such a structure to process log files?</p>
<p><strong>Stacks of log</strong></p>
<p>On a company ecosystem, there is lots of systems, like the CRM, ERP, etc. On such environments, it is common for the systems to produce tons of logs, which provide not only a real-time analysis of the technical status of the software, but could also provide some business information too, like a log of a customer's behavior on a shopping cart, for example. To dive into this useful source of information, enters the ELK architecture, which name came from the initials of the software involved: ElasticSearch, LogStash and Kibana. The picture below shows in a macro vision the flow between the tools:</p>
<p><img src="https://dl.dropboxusercontent.com/s/4kqrjggb1vqfqhq/architectureELKdzone.jpg?dl=0" alt=""/></p>
<p>As we can see, there's a clear separation of concerns between the tools, where which one has his own individual part on the processing of the log data:</p>
<ul>
<li> <strong>Logstash</strong>: Responsible for collect the data, make transformations like parsing - using regular expressions - adding fields, formatting as structures like JSON, etc and finally sending the data to various destinations, like a ElasticSearch cluster. Later on this post we will see more detail about this useful tool;</li>
<li> <strong>ElasticSearch</strong>: RESTful data indexer, ElasticSearch provides a clustered solution to make searches and analysis on a set of data. On the second part of our series, we will see more about this tool;</li>
<li> <strong>Kibana</strong>: Web-based application, responsible for providing a light and easy-to-use dashboard tool. On the third and last part of our series, we will see more of this tool;</li>
</ul>
<p>So, to begin our road in the ELK stack, let's begin by talking about the tool responsible for integrating our data: LogStash.</p>
<p><strong>LogStash installation</strong></p>
<p>To install, all we need to do is unzip the file we get from LogStash's site and run the binaries on the bin folder. The only pre-requisite for the tool is to have Java installed and configured in the environment. If the reader wants to follow my instructions with the same system then me, I am using Ubuntu 14.10 with Java 8, which can be downloaded from Oracle's site <a href="http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html">here</a>.</p>
<p>With Java installed and configured, we begin by downloading and unziping the file. To do this, we open a terminal and input:</p>
<pre><code class="language-sh">curl https://download.elasticsearch.org/logstash/logstash/logstash-1.4.2.tar.gz | tar -xz
</code></pre>
<p>After the download, we will have LogStash on a folder on the same place we run our 'curl' command. On the LogStash terminology, we have 4 types of configurations we can make for a stream, named:</p>
<ul>
<li> <strong>input</strong>: On this configuration, we put the sources of our streams, that can range from polling files of a file system to more complex inputs such as a Amazon SQS queue and even Twitter;</li>
<li> <strong>codec</strong>: On this configuration we make transformations on the data, like turning into a JSON structure, or grouping together lines that are semantically related, like for example, a Java's stack trace;</li>
<li> <strong>filter</strong>: On this configuration we make operations such as parsing data from/to different formats, removal of special characters and checksums for deduplication;</li>
<li> <strong>output</strong>: On this configuration we define the destinations for the processed data, such as a ElasticSearch cluster, AWS SQS, Nagios etc;</li>
</ul>
<p>Now that we have established LogStash's configuration structure, let's begin with our first execution. In LogStash we have two ways to configure our execution, one way by providing the settings on the start command itself and the other one is by providing a configuration file for the command. The simplest way to boot a LogStash's stream is by setting the input and output as the console itself, to make this execution, we open a terminal, navigate to the bin folder of our LogStash's installation and execute the following command:</p>
<pre>./logstash -e 'input { stdin { } } output { stdout {} }'</pre>
<p>As we can see after we run the command, we booted LogStash, setting the console as the input and the output, without any transformation or filtering. To test, we simply input anything on the console, seeing that our message is displayed back by the tool:</p>
<p><img src="https://dl.dropboxusercontent.com/s/c2s4wiuo585hxaj/ELKpart1image1.jpg?dl=0" alt=""/></p>
<p>Now that we get the installation out of the way, let's begin with the actual lab. Unfortunately -or not, depending on the point of view -, it would take us a lot of time to show all the features of what we can do with the tool, so to make a short but illustrative example, we will start 2 logstash streams, to do the following:</p>
<p>1st stream:</p>
<ul>
<li> The input will be made by a java program, which will produce a log file with log4j, representing technical information;</li>
<li> For now, we will just print logstash's events on the console, using the rubydebug codec. On our next part on the series, we will return to this configuration and change the output to send the events to elasticsearch;</li>
</ul>
<p>2nd stream:</p>
<ul>
<li> The input will be made by the same java program, which will produce a positional file, representing business information of costumers and orders;</li>
<li> We will then use the grok filter to parse the data of the positional file into separated fields, producing the data for the output step;</li>
<li> Finally, we use the mongodb output, to save our data - filtering to only persist the orders - on a Mongodb collection;</li>
</ul>
<p>With the streams defined, we can begin our coding. First, let's create the java program which will generate the inputs for the streams. The code for the program can be seen bellow:</p>
<pre><code class="language-java">package com.technology.alexandreesl;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.log4j.Logger;
public class LogStashProvider {
private static Logger logger = Logger.getLogger(LogStashProvider.class);
public static void main(String[] args) throws IOException {
try {
logger.info("STARTING DATA COLLECTION");
List<String> data = new ArrayList<String>();
Customer customer = new Customer();
customer.setName("Alexandre");
customer.setAge(32);
customer.setSex('M');
customer.setIdentification("4434554567");
List<Order> orders = new ArrayList<Order>();
for (int counter = 1; counter < 10; counter++) {
Order order = new Order();
order.setOrderId(counter);
order.setProductId(counter);
order.setCustomerId(customer.getIdentification());
order.setQuantity(counter);
orders.add(order);
}
logger.info("FETCHING RESULTS INTO DESTINATION");
PrintWriter file = new PrintWriter(new FileWriter(
"/home/alexandreesl/logstashdataexample/data"
+ new Date().getTime() + ".txt"));
file.println("1" + customer.getName() + customer.getSex()
+ customer.getAge() + customer.getIdentification());
for (Order order : orders) {
file.println("2" + order.getOrderId() + order.getCustomerId()
+ order.getProductId() + order.getQuantity());
}
logger.info("CLEANING UP!");
file.flush();
file.close();
// forcing a error to simulate stack traces
PrintWriter fileError = new PrintWriter(new FileWriter(
"/etc/nopermission.txt"));
} catch (Exception e) {
logger.error("ERROR!", e);
}
}
}
</code></pre>
<p>As we can see, it is a very simple class, that uses log4j to generate some log and output a positional file representing data from customers and orders and at the end, try to create a file on a folder we don't have permission to write by default,"forcing" a error to produce a stack trace. The complete code for the program can be found <a href="https://github.com/alexandreesl/LogStashProvider.git">here</a>. Now that we have made our data generator, let's begin the configuration for logstash. The configuration for our first example is the following:</p>
<pre><code class="language-conf">input {
log4j {
port => 1500
type => "log4j"
tags => [ "technical", "log"]
}
}
output {
stdout { codec => rubydebug }
}
</code></pre>
<p>To run the script, let's create a file called"config1.conf"and save the file with the script on the"bin"folder of logstash's installation folder. Finally, we run the script with the following command:</p>
<pre><code class="language-sh">./logstash -f config1.conf
</code></pre>
<p>This will start logstash process with the configurations we provided. To test, simply run the java program we coded earlier and we will see a sequence of message events in logstash's console window, generated by the rubydebug codec, like the one bellow, for example:</p>
<pre><code class="language-log">{
"message" => "ERROR!",
"@version" => "1",
"@timestamp" => "2015-01-24T19:08:10.872Z",
"type" => "log4j",
"tags" => [
[0] "technical",
[1] "log"
],
"host" => "127.0.0.1:34412",
"path" => "com.technology.alexandreesl.LogStashProvider",
"priority" => "ERROR",
"logger_name" => "com.technology.alexandreesl.LogStashProvider",
"thread" => "main",
"class" => "com.technology.alexandreesl.LogStashProvider",
"file" => "LogStashProvider.java:70",
"method" => "main",
"stack_trace" => "java.io.FileNotFoundException: /etc/nopermission.txt (Permission denied)\n\tat java.io.FileOutputStream.open(Native Method)\n\tat java.io.FileOutputStream.<init>(FileOutputStream.java:213)\n\tat java.io.FileOutputStream.<init>(FileOutputStream.java:101)\n\tat java.io.FileWriter.<init>(FileWriter.java:63)\n\tat com.technology.alexandreesl.LogStashProvider.main(LogStashProvider.java:66)"
}
</code></pre>
<p>Now, let's move on to the next stream. First, we create another file, called"config2.conf", on the same folder we created the first one. On this new file, we create the following configuration:</p>
<pre><code class="language-conf">input {
file {
path = > "/home/alexandreesl/logstashdataexample/data*.txt"
start_position = > "beginning"
}
}
filter {
grok {
match = > ["message", "(?<file_type>.{1})(?<name>.{9})(?<sex>.{1})(?<age>.{2})(?<identification>.{10})", "message", "(?<file_type>.{1})(?<order_id>.{1})(?<costumer_id>.{10})(?<product_id>.{1})(?<quantity>.{1})"]
}
}
output {
stdout {codec = > rubydebug}
if [file_type] == "2" {
mongodb {
collection = > "testData"
database = > "mydb"
uri = > "mongodb://localhost"
}
}
}
</code></pre>
<p>With the configuration created, we can run our second example. Before we do that, however, let's dive a little on the configuration we just made. First, we used the file input, which will make logstash keep monitoring the files on the folder and processing them as they appear on the input folder.</p>
<p>Next, we create a filter with the grok plugin. This filter uses combinations of regular expressions, that parses the data from the input. The plugin comes with more then 100 patterns pre-made that helps the development. Another useful tool in the use of grok is a site where we could test our expressions before use. Both links are available on the links section at the end of this post.</p>
<p>Finally, we use the mongodb plugin, where we reference our logstash for a database and collection of a mongodb instance, where we will insert the data from the file into mongodb's documents. We also used again the rubydebug codec, so we can also see the processing of the files on the console. The reader will note that we used a"if"statement before the configuration of the mongodb output. After we parse the data with grok, we can use the newly created fields to do some logic on our stream. In this case, we filter to only process data with the type"2", so only the order's data goes to the collection on mongodb, instead of all the data. We could have expanded more on this example, like saving the data into two different collections, but for the idea of passing a general view of the structure of logstash for the reader, the present logic will suffice.</p>
<p><strong>PS:</strong> This example assumes the reader has mongodb installed and running on the default port of his environment, with a db "mydb" and a collection "testData" created. If the reader doesn't have mongodb, the instructions can be found on the <a href="http://docs.mongodb.org/manual/tutorial/getting-started/">official documentation</a>.</p>
<p>Finally, with everything installed and configured, we run the script, with the following command:</p>
<pre>./logstash -f config2.conf</pre>
<p>After logstash's start, if we run our program to generate a file, we will see logstash working the data, like the screen bellow:</p>
<p><img src="https://dl.dropboxusercontent.com/s/sjh61dwr5j50t1e/logstash2.png?dl=0" alt=""/></p>
<p>And finally, if we query the collection on mongodb, we see the data is persisted:</p>
<p><img src="https://dl.dropboxusercontent.com/s/2rk8e45qly5glkf/logstash3.png?dl=0" alt=""/></p>
<p><strong>Conclusion</strong></p>
<p>And so we conclude the first part of our series. With a simple usage, logstash prove to be a useful tool in the integration of information from different formats and sources, specially log-related. In the next part of our series, we will dive in the next tool of our stack: ElasticSearch. Until next time</p>
<p><a href="http://logstash.net/">logstash - official site</a></p>
<p><a href="https://github.com/alexandreesl/LogStashProvider.git">Source-code (Github)</a></p>
<p><a href="https://grokdebug.herokuapp.com/">Grok Debugger (online testing of grok expressions)</a></p>
<p><a href="https://github.com/elasticsearch/logstash/blob/v1.4.2/patterns/grok-patterns">Grok pre-made patterns</a></p>
<hr/>
<p>Welcome, dear reader, to another post of our series about the ELK stack for logging. On the last post, we talked about LogStash, a tool that allow us to integrate data from different sources to different destinations, using transformations along the way, in a stream-like form. On this post, we will talk about ElasticSearch, a indexer based on apache Lucene, which can allow us to organize our data and make textual searches on the data, in a scalable infrastructure. So, let's begin by understanding how ElasticSearch is organized on the inside</p>
<p><strong>Indexes, documents and shards</strong></p>
<p>On ElasticSearch, we have the concept of indexes. A index is like a repository, where we can store our data in the format of documents. A document on ElasticSearch's terminology consists of a structure for the data to be stored, analysed and classified, following a mapping definition, composed of a series of fields - a important thing to note, is that a field on ElasticSearch has the same type across the whole index, meaning that we cant have a field"phone" with the type int on a document and the type string on another.</p>
<p>In turn, we have our documents stored on shards, which divide the data on segments based on a rule - by default, the segmentation is made by hashing the data, but it can also be manually manipulated -, making the searches faster.</p>
<p>So, in a nutshell, we can say that the order of organization of ElasticSearch is as follows:</p>
<p>Index >> Document (mappings/type) >> shard</p>
<p>This organization is used by the user on the two basic operations of the cluster: indexing and searching.</p>
<p>One last thing to say about documents is that they can not only be stored as independent , but also be mounted on a tree-like hierarchy, with links between them. This is useful in scenarios that we can make use of hierarchical searches, such as product's searches based on their categories.</p>
<p><strong>Indexing</strong></p>
<p>Indexing is the action of inputing the data from a external source to the cluster. ElasticSearch is a textual indexer, which means he can only analyse text on plain format, despite that we can use the cluster to store data in base64 format, using a plugin. Later on the post, we will see a example installation of a plugin, which are extensions we can aggregate to expand our cluster usability.</p>
<p>When we index our data, we define which fields are to be analysed, which analyser to use, if the default ones does not suffice and which fields we want to store the data on the cluster, so we can use as the result of our searches. One important thing to note about the indexing operations is that, despite it has CRUD-like operations, the data is not really updated or deleted on the cluster, instead a new version is generated and the old version is marked as deleted.</p>
<p>This is a important thing to take note, because if not properly configured to make purges - which can be made with a configuration that break the shards into segments, and periodically make merges of the segments, phisically deleting the obsolet documents on the process -, the cluster will keep indefinitely expanding in size with the "deleted" older versions of our data, making specially the searches to became really slow.</p>
<p>All the operations can be made with a REST API provided by ElasticSearch, that we will see later on this post.</p>
<p><strong>Searching</strong></p>
<p>The other, and probably most important, action on ElasticSearch, is the searching of the data previously indexed. Like the indexing action, ElasticSearch also provide a REST API for the searches. The API provides a very rich range of possibilities of searching, from basic term searches to more complex searches such as hierarchical searches, searches by synonims, language detections, etc.</p>
<p>All the searching is based on a score system, where formulas are applied to confront the accuracy of the documents founded versus the query supplied. This score system can also be customized.</p>
<p>By default, the searching on the cluster occurs in 2 phases:</p>
<ul>
<li> On the first phase, the master node sends the query for all the nodes, and subsequently shards , retrieving just the IDs and scores of the documents. Using a parameter called _size_ which defines the maximum results from a query, the master selects the more meaningful documents, based on the score;</li>
<li> On the second phase, the master send requests for the nodes to retrieve the documents selected on the previous phase. After receiving the documents, the master finally sends the result for the client;</li>
</ul>
<p>Alongside this search type, there's also other modes, like the _query_and_fetch_. On this mode, the searching is made simultaneous on all shards, not only to retrieve the IDs and scores but also returning the data itself, limited only by the _size_ parameter, which is applied per shard. In turn, on this mode, the maximum of results returned will be the size parameter plus the number of shards.</p>
<p>One interesting feature of ElasticSearch's configuration options is the ability to make some nodes exclusive to query operations, and others to make the storage part, called data nodes. This way, when we query, our query dont need to run across all the cluster to formulate the results, making the searches faster. On the next section we will see a little more about cluster configurations.</p>
<p><strong>Cluster capabilities</strong></p>
<p>When we talk about a cluster, we talk about scalability, but we also talk about availability. On ElasticSearch, we can configure the replication of shards, where the data is replicated by a given factor, so we dont lose our data if a node is lost. The replication if also maintained by the cluster, so if we lost a replica, the cluster itself will distribute a new replica for another node.</p>
<p>Other interesting feature of the cluster are the ability to discover itself. By the default configuration, when we start a node he will use a discovery mode called Zen, which uses unicast and multicast to search for another instances on all the ports of the OS. If it founds another instance, and the name of the cluster is the same - this is another one of the cluster's configuration properties. All of this configurations can be made on the file _elasticsearch.yml_, on the config folder -, it will communicate with the instance and establish a new node for the already running cluster. There is another modes for this feature, including the discover of nodes from other servers.</p>
<p><strong>Logging</strong></p>
<p>The reader could be thinking: "Lol, do I need all of this to run a logging stack?".</p>
<p>Of course that ElasticSearch is a very robust tool, that can be used on other solutions as well. However, on our case of making a centralized logging analysis solution, the core of ElasticSearch's capabilities serve us well for this task, after all, we are talking about the textual analysis of log texts, for use on dashboards, reports, or simply for real-time exploration of the data.</p>
<p>Well, that concludes the conceptual part of our post. Now, let's move on to the practice.</p>
<p><strong>Hands-on</strong></p>
<p>So, without further delay, let's begin the hands-on. For this, we will use the previous Java program we used on our lab about LogStash. The code can be found on GitHub, <a href="https://github.com/alexandreesl/LogStashProvider.git">on this link</a>. On this program, we used the _org.apache.log4j.net.SocketAppender_ from log4j to send all the logging we make to LogStash. However, on that point we just printed the messages on the console, instead of sending to ElasticSearch. Before we change this, let's first start our cluster.</p>
<p>To do this, first we need to download the last version from the site and unzip the tar. Let's open a terminal, and type the following command:</p>
<pre>curl https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.4.tar.gz | tar -zx</pre>
<p>After running the command, we will find a new folder called "elasticsearch-1.4.4" created on the same folder we run our command. To our example, we will create 2 copies of this folder on a folder we call "elasticsearchcluster", where each one will represent one node of the cluster. To do this, we run the following commands:</p>
<pre><code class="language-sh">mkdir elasticsearchcluster
sudo cp -avr elasticsearch-1.4.4/ elasticsearchcluster/elasticsearch-1.4.4-node1/
sudo cp -avr elasticsearch-1.4.4/ elasticsearchcluster/elasticsearch-1.4.4-node2/
</code></pre>
<p>After we made our cluster structure, we dont need the original folder anymore, so we remove:</p>
<pre><code class="language-sh">rm -R elasticsearch-1.4.4/
</code></pre>
<p>Now, let's finally start our cluster! To do this, we open a terminal, navigate to the bin folder of our first node (elasticsearch-1.4.4-node1) and type:<a href="#viewSource" title="view source">view sourc</a></p>
<pre><code class="language-sh">./elasticsearch
</code></pre>
<p>After some seconds, we can see our first node is on:</p>
<p><img src="https://dl.dropboxusercontent.com/s/0mb9no6kee8oicq/elastic1.png?dl=0" alt=""/></p>
<p>For curiosity sake, we can see the name "Feral" on the node's name on the log. All the names generated by the tool are based on Marvel Comic's characters. IT world sure has some sense of humor, heh?</p>
<p>Now, let's start our second node. On a new terminal window, let's navigate to the folder of our second node (elasticsearch-1.4.4-node2) and type again the command "./elasticsearch". After some seconds, we can see that the node is also started:</p>
<p><img src="https://dl.dropboxusercontent.com/s/816wtb4ak0o1j3b/elastic2.png?dl=0" alt=""/></p>
<p>One interesting thing to notice is that our second node "Ooze", has a mention of comunicating with our other node, "Feral". That is the zen discover on the action, making the 2 nodes talk to each other and form a cluster. If we look again at the terminal of our first node, we can see another evidence of this bidirectional communication, as "Feral" has added "Ooze" to the cluster, as his role as a master node:</p>
<p><img src="https://dl.dropboxusercontent.com/s/p15tuaw4qewmz2i/elastic3.png?dl=0" alt=""/></p>
<p> Now that we have our cluster set up, let's adjust our logstash script to send the messages to the cluster. To do this, let's change the output part of the script, to the following:</p>
<pre><code class="language-conf">input {
log4j {
port = > 1500
type = > "log4j"
tags = > ["technical", "log"]
}
}
output {
stdout {codec = > rubydebug}
elasticsearch_http {
host = > "localhost"
port = > 9200
index = > "log4jlogs"
}
}
</code></pre>
<p>As we can see, we just included another output - we remained the console output just to check how logstash is receiving the data - including the ip and port where our ElasticSearch cluster will respond. We also defined the name of the index we want our logs to be stored. If this parameter is not defined, logstash will order elasticsearch to create a index with the pattern "logstash-%{+YYYY.MM.dd}".</p>
<p>To execute this script, we do like we did on the <a href="https://alexandreesl.wordpress.com/2015/01/26/elk-using-a-centralized-logging-architecture-part-1/" title="ELK: using a centralized logging architecture – part 1">previous post</a>, we put the new script on a file called "configelasticsearch.conf" on the bin folder of logstash, and run with the command:</p>
<pre><code class="language-sh">./logstash -f configelasticsearch.conf
</code></pre>
<p><strong>PS1:</strong> On the<a href="https://github.com/alexandreesl/ElasticSearchConfigs.git"> GitHub repository</a>, it is possible to find this config file, alongside a file containing all the commands we will send to ElasticSearch from now on.</p>
<p><strong>PS2: </strong>For simplicity sake, we will use the default mappings logstash provide for us when sending messages to the cluster. It is also possible to pass a elasticsearch's mapping structure, which consists of a JSON model, that logstash will use as a template. We will see the mapping from our log messages later on our lab, but for satisfying the reader curiosity for now, this is what a elasticsearch's mapping structure look like, for example for a document type "product":</p>
<pre><code class="language-json">"mappings": {
"product": {
"properties": {
"variation": {"type": int}
"color": {"type": "string"}
"code": {"type": int}
"quantity": {"type": int}
}
}
}
</code></pre>
<p>After some seconds, we can see that LogStash booted, so our configuration was a success. Now, let's begin sending our logs!</p>
<p>To do this, we run the program from our previous post, running the class _com.technology.alexandreesl.LogStashProvider ._ We can see on the console of logstash, after starting the program, that the messages are going through the stack:</p>
<p><img src="https://dl.dropboxusercontent.com/s/avoh8yu98u5oqbj/elastic4.png?dl=0" alt=""/></p>
<p>Now that we have our cluster up and running, let's start to use it. First, let's see the mappings of the index that ElasticSearch created for us, based on the configuration we made on LogStash. Let's open a terminal and run the following command:</p>
<p>curl -XGET 'localhost:9200/log4jlogs/_mapping?pretty'</p>
<p>On the command above, we are using ElasticSearch's REST API. The reader will notice that, after the ip and port, the URL contains the name of the index we configured. This pattern for calls of the API is applied to most of the actions, as we can see below:</p>
<p><ip>:<port>/<index>/<doc type>/<action>?<attributes></p>
<p>So, after this explanation, let's see the result from our call:</p>
<pre><code class="language-json">{
"log4jlogs": {
"mappings": {
"log4j": {
"properties": {
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
},
"@version": {
"type": "string"
},
"class": {
"type": "string"
},
"file": {
"type": "string"
},
"host": {
"type": "string"
},
"logger_name": {
"type": "string"
},
"message": {
"type": "string"
},
"method": {
"type": "string"
},
"path": {
"type": "string"
},
"priority": {
"type": "string"
},
"stack_trace": {
"type": "string"
},
"tags": {
"type": "string"
},
"thread": {
"type": "string"
},
"type": {
"type": "string"
}
}
}
}
}
}
</code></pre>
<p>As we can see, the index "log4jlogs" was created, alongside the document type "log4j". Also, a series of fields were created, representing information from the log messages, like the thread that generated the log, the class, the log level and the log message itself.</p>
<p>Now, let's begin to make some searches.</p>
<p>Let's begin by searching all log messages which the priority was"INFO". We make this searching by running:</p>
<p>curl -XGET 'localhost:9200/log4jlogs/log4j/_search?q=priority:info&pretty=true'</p>
<p>A fragment of the result of the query would be something like the following:</p>
<pre><code class="language-json">{
"took": 12,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 18,
"max_score": 1.1823215,
"hits": [{
"_index": "log4jlogs",
"_type": "log4j",
"_id": "AUuxkDTk8qbJts0_16ph",
"_score": 1.1823215,
"_source": {"message": "STARTING DATA COLLECTION", "@version": "1", "@timestamp": "2015-02-22T13:53:12.907Z", "type": "log4j", "tags": ["technical", "log"], "host":"127.0.0.1:32942", "path":"com.technology.alexandreesl.LogStashProvider", "priority":"INFO", "logger_name":"com.technology.alexandreesl.LogStashProvider", "thread":"main", "class":"com.technology.alexandreesl.LogStashProvider", "file":"LogStashProvider.java:20", "method":"main"}
}
.
.
.
</code></pre>
<p>As we can see, the result is a JSON structure, with the documents that met our search. The beginning information of the result is not the documents themselves, but instead information about the search itself, such as the number of shards used, the seconds the search took to execute, etc. This kind of information is useful when we need to make a tuning of our searches, like manually defining the shards we which to use on the search, for example.</p>
<p>Let's see another example. On our previous search, we received all the fields from the document on the result, which is not always the desired result, since we will not always use the whole information. To limit the fields we want to receive, we make our query like the following:</p>
<pre><code class="language-sh">curl -XGET 'localhost:9200/log4jlogs/log4j/_search?pretty=true' -d '
{
"fields" : ["priority", "message","class"],
"query" : {
"query_string" : {"query" : "priority:info"}
}
}'
</code></pre>
<p>On the query above, we asked ElasticSearch to limit the return to only return the priority, message and class fields. A fragment of the result can be seen bellow:</p>
<pre><code class="language-json">.
.
.
{
"_index" : "log4jlogs",
"_type" : "log4j",
"_id" : "AUuxkECZ8qbJts0_16pr",
"_score" : 1.1823215,
"fields" : {
"priority" : [ "INFO" ],
"message" : [ "CLEANING UP!" ],
"class" : [ "com.technology.alexandreesl.LogStashProvider" ]
}
}
.
.
.
</code></pre>
<p>Now, let's use the term search. On the term searches, we use ElasticSearch's textual analysis to find a term inside the text of a field. Let's run the following command:</p>
<pre><code class="language-sh">curl -XGET 'localhost:9200/log4jlogs/log4j/_search?pretty=true' -d '
{
"fields" : ["priority", "message","class"],
"query" : {
"term" : {
"message" : "up"
}
}
}'
</code></pre>
<p>If we see the result, it would be all the log messages that contains the word "up". A fragment of the result can be seen bellow:</p>
<pre><code class="language-json">{
"_index" : "log4jlogs",
"_type" : "log4j",
"_id" : "AUuxkESc8qbJts0_16pv",
"_score" : 1.1545612,
"fields" : {
"priority" : [ "INFO" ],
"message" : [ "CLEANING UP!" ],
"class" : [ "com.technology.alexandreesl.LogStashProvider" ]
}
}
</code></pre>
<p>Of course, there is a lot more of searching options on ElasticSearch, but the examples provided on this post are enough to make a good starting point for the reader. To make a final example, we will use the "prefix" search. On this type of search, ElasticSearch will search for terms that start with our given text, on a given field. For example, to search for log messages that have words starting with "clea", part of the word "cleaning", we run the following:</p>
<pre><code class="language-sh">curl -XGET 'localhost:9200/log4jlogs/log4j/_search?pretty=true' -d '
{
"fields" : ["priority", "message","class"],
"query" : {
"prefix" : {
"message" : "clea"
}
}
}'
</code></pre>
<p>If we see the results, we will see that are the same from the previous search, proving that our search worked correctly.</p>
<p><strong>Kopf</strong></p>
<p>The reader possibly could ask "Is there another way to send my queries without using the terminal?" or "Is there any graphical tool that I can use to monitor the status of my cluster?". As a matter of fact, there is a answer for both of this questions, and the answer is the kopf plugin.</p>
<p>As we said before, plugins are extensions that we can install to improve the capacities of our cluster. In order to install the plugin, first let's stop both the nodes of the cluster - press ctrl+c on both terminal windows to stop - then, navigate to the nodes root folder and type the following:</p>
<pre><code class="language-sh">bin/plugin -install lmenezes/elasticsearch-kopf
</code></pre>
<p>If the plugin was installed correctly, we should see a message like the one bellow on the console:</p>
<pre><code class="language-log">.
.
.
-> Installing lmenezes/elasticsearch-kopf...
Trying [https://github.com/lmenezes/elasticsearch-kopf/archive/master.zip...](https://github.com/lmenezes/elasticsearch-kopf/archive/master.zip...)
Downloading .....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................DONE
Installed lmenezes/elasticsearch-kopf into....
</code></pre>
<p>After installing on both nodes, we can start again the nodes, just as we did before. After the booting of the cluster, let's open a browser and type the following URL:</p>
<p><a href="http://localhost:9200/_plugin/kopf">http://localhost:9200/_plugin/kopf</a></p>
<p>We will see the following web page of the kopf plugin, showing the status of our cluster, such as the nodes, the indexes, shard information, etc</p>
<p><img src="https://dl.dropboxusercontent.com/s/v1gevf01knh4efo/elastic5.png?dl=0" alt=""/></p>
<p>Now, let's run our last example from the search queries on kopf. First, we select the"rest"option on the top menu. On the next screen, we select"POST" as the http method, include on the URL field the index and document type to narrow the results and on the textarea bellow we include our JSON query filters. The print bellow shows the query made on the interface:</p>
<p><img src="https://dl.dropboxusercontent.com/s/qpc23126qwkakpy/elastic6.png?dl=0" alt=""/></p>
<p><strong> Conclusion</strong></p>
<p>And so we conclude our post about ElasticSearch. A very powerful tool on the indexing and analysis of textual information, the central stone on our ELK stack for logging is a tool to be used, not only on a logging analysis system, but on other solutions that his features can be useful as well.</p>
<p>So, our stack is almost complete. We can gather our log information, and the information is indexed on our cluster. However, a final piece remains: we need a place where we can have a more friendly interface, that allow us not only to search the information, but also to make rich presentations of the data, such as dashboards. That's when it enters our last part of our ELK series and the last tool we will see, Kibana. Thank you for following me on another post, until next time.</p>
<p><a href="https://github.com/alexandreesl/ElasticSearchConfigs">GitHub code (ElasticSearch)</a></p>
<p><a href="https://github.com/alexandreesl/LogStashProvider">GitHub code (LogStash)</a></p>
<p><a href="http://www.elasticsearch.org/">ElasticSearch (Official site)</a></p>
</div>
<div class="row">
<div class="large-6 columns">
<p class="text-left" style="padding:15px 0px;">
<a href="15282782041132.html"
title="Previous Post: Enabling Centralized Logging">« Enabling Centralized Logging</a>
</p>
</div>
<div class="large-6 columns">
<p class="text-right" style="padding:15px 0px;">
<a href="15282768663638.html"
title="Next Post: Part 1: Building a Centralized Logging Application">Part 1: Building a Centralized Logging Application »</a>
</p>
</div>
</div>
<div class="comments-wrap">
<div class="share-comments">
<script type="text/javascript" src="//s7.addthis.com/js/300/addthis_widget.js#pubid=ra-5ae58078c0d7b2ab"></script>
</div>
</div>
</div><!-- article-wrap -->
</div><!-- large 8 -->
<div class="large-4 medium-4 columns">
<div class="hide-for-small">
<div id="sidebar" class="sidebar">
<div id="site-info" class="site-info">
<div class="site-a-logo"><img src="./asset/img/logo.jpg" /></div>
<h1>Junkman</h1>
<div class="site-des">“拾荒者”一词来自凯文・凯利的《失控》中关于机器学习的故事（“收集癖好机”如何完成他的收集工作）。</div>
<div class="social">
<a target="_blank" class="github" target="_blank" href="https://github.com/panlw/" title="GitHub">GitHub</a>
<a target="_blank" class="rss" href="atom.xml" title="RSS">RSS</a>
</div>
</div>
<div id="site-categories" class="side-item ">
<div class="side-header">
<h2>Categories</h2>
</div>
<div class="side-content">
<p class="cat-list">
<a href="Infra.html"><strong>Infra</strong></a>
<a href="Coding.html"><strong>Coding</strong></a>
<a href="Modeling.html"><strong>Modeling</strong></a>
<a href="Archtecting.html"><strong>Archtecting</strong></a>
</p>
</div>
</div>
<div id="site-categories" class="side-item">
<div class="side-header">
<h2>Recent Posts</h2>
</div>
<div class="side-content">
<ul class="posts-list">
<li class="post">
<a href="15517999043443.html">The Art of Crafting Architectural Diagrams</a>
</li>
<li class="post">
<a href="15517997955971.html">为什么说我们需要软件架构图？</a>
</li>
<li class="post">
<a href="15516128677869.html">DNS Servers That Offer Privacy and Filtering</a>
</li>
<li class="post">
<a href="15516123108194.html">Airbnb's Migration from Monolith to Services</a>
</li>
<li class="post">
<a href="15516097487470.html">Events As First-Class Citizens</a>
</li>
</ul>
</div>
</div>
</div><!-- sidebar -->
</div><!-- hide for small -->
</div><!-- large 4 -->
</div><!-- row -->
<div class="page-bottom clearfix">
<div class="row">
<p class="copyright">Copyright © 2015
Powered by <a target="_blank" href="http://www.mweb.im">MWeb</a>,
Theme used <a target="_blank" href="http://github.com">GitHub CSS</a>.</p>
</div>
</div>
</section>
</div>
</div>
<script src="asset/js/foundation.min.js"></script>
<script>
$(document).foundation();
function fixSidebarHeight(){
var w1 = $('.markdown-body').height();
var w2 = $('#sidebar').height();
if (w1 > w2) { $('#sidebar').height(w1); };
}
$(function(){
fixSidebarHeight();
})
$(window).load(function(){
fixSidebarHeight();
});
</script>
<script src="asset/chart/all-min.js"></script><script type="text/javascript">$(function(){ var mwebii=0; var mwebChartEleId = 'mweb-chart-ele-'; $('pre>code').each(function(){ mwebii++; var eleiid = mwebChartEleId+mwebii; if($(this).hasClass('language-sequence')){ var ele = $(this).addClass('nohighlight').parent(); $('<div id="'+eleiid+'"></div>').insertAfter(ele); ele.hide(); var diagram = Diagram.parse($(this).text()); diagram.drawSVG(eleiid,{theme: 'simple'}); }else if($(this).hasClass('language-flow')){ var ele = $(this).addClass('nohighlight').parent(); $('<div id="'+eleiid+'"></div>').insertAfter(ele); ele.hide(); var diagram = flowchart.parse($(this).text()); diagram.drawSVG(eleiid); } });});</script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script type="text/x-mathjax-config">MathJax.Hub.Config({TeX: { equationNumbers: { autoNumber: "AMS" } }});</script>
</body>
</html>