Skip to content

Commit

Permalink
Merge pull request #466 from Meituan-Dianping/refactor/server_decoder
Browse files Browse the repository at this point in the history 
后端日志解析优化，解决日志丢失问题
  • Loading branch information
@Richard-Cao
Richard-Cao authored Mar 1, 2023
2 parents b19012f + 8f8c33f commit 3a81b56
Show file tree
Hide file tree
Showing 4 changed files with 264 additions and 21 deletions.
6 changes: 6 additions & 0 deletions Logan/Clogan/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
cmake_minimum_required(VERSION 3.5)

project(clogan)

set(MBEDTSL_INCLUDE ../mbedtls/include)
set(MBEDTSL_LIBRARY ../mbedtls/library)

Expand All @@ -18,3 +20,7 @@ set(SOURCE_FILES

add_library (clogan ${SOURCE_FILES})
target_link_libraries(clogan mbedcrypto)
target_link_libraries(clogan z)

add_executable(clogan_runner main.c)
target_link_libraries(clogan_runner clogan)
37 changes: 37 additions & 0 deletions Logan/Clogan/main.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#include "base_util.h"
#include "clogan_core.h"
#include <stdio.h>
#include <time.h>

int main(int argc, const char **argv) {
char key[16] = "0123456789012345";
char iv[16] = "0123456789012345";
int code;

clogan_debug(0);
code = clogan_init("log", "log", 10 * 1024 * 1024, key, iv);
printf("[init]: %d\n", code);

code = clogan_open("logan.bin");
printf("[open]: %d\n", code);

long long ts = get_system_current_clogan();
time_t now = time(NULL);
char *now_str = ctime(&now);
printf("[now]: %s\n", now_str);

clogan_flush();
clogan_write(10, now_str, ts, "main", 1, 1);
clogan_write(10, "[log 1]", ts++, "main", 1, 1);
clogan_write(10, "[log 2]", ts++, "main", 1, 1);
clogan_write(10, "[log 3]", ts++, "main", 1, 1);
clogan_write(10, "[log 4]", ts++, "main", 1, 1);
clogan_write(10, "[log 5]", ts++, "main", 1, 1);
clogan_write(10, "how", ts++, "main", 1, 1);
clogan_write(10, "are", ts++, "main", 1, 1);
clogan_write(10, "you", ts++, "main", 1, 1);
clogan_write(10, "fine", ts++, "main", 1, 1);
// clogan_flush();

printf("[exit]\n");
}
136 changes: 136 additions & 0 deletions Logan/Server/src/main/java/com/meituan/logan/web/parser/LegacyLoganProtocol.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
package com.meituan.logan.web.parser;

import com.meituan.logan.web.enums.ResultEnum;
import com.meituan.logan.web.model.Tuple;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.springframework.core.io.support.PropertiesLoaderUtils;

import javax.annotation.PreDestroy;
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.*;
import java.nio.ByteBuffer;
import java.security.Security;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.GZIPInputStream;

/**
* 老版本解析器
*/
public class LegacyLoganProtocol {

private static final Logger LOGGER = Logger.getLogger(LoganProtocol.class);

private static final char ENCRYPT_CONTENT_START = '\1';

private static final String AES_ALGORITHM_TYPE = "AES/CBC/NoPadding";

private static AtomicBoolean initialized = new AtomicBoolean(false);

static {
initialize();
}

private ByteBuffer wrap;
private FileOutputStream fileOutputStream;

public LegacyLoganProtocol(InputStream stream, File file) {
try {
wrap = ByteBuffer.wrap(IOUtils.toByteArray(stream));
fileOutputStream = new FileOutputStream(file);
} catch (IOException e) {
LOGGER.error(e);
}
}

public ResultEnum process() {
while (wrap.hasRemaining()) {
while (wrap.hasRemaining() && wrap.get() == ENCRYPT_CONTENT_START) {
byte[] encrypt = new byte[wrap.getInt()];
if (!tryGetEncryptContent(encrypt) || !decryptAndAppendFile(encrypt)) {
return ResultEnum.ERROR_DECRYPT;
}
}
}
return ResultEnum.SUCCESS;
}

private boolean tryGetEncryptContent(byte[] encrypt) {
try {
wrap.get(encrypt);
} catch (java.nio.BufferUnderflowException e) {
LOGGER.error(e);
return false;
}
return true;
}

private boolean decryptAndAppendFile(byte[] encrypt) {
boolean result = false;
try {
Cipher aesEncryptCipher = Cipher.getInstance(AES_ALGORITHM_TYPE);
Tuple<String, String> secureParam = getSecureParam();
if (secureParam == null) {
return false;
}
SecretKeySpec secretKeySpec = new SecretKeySpec(secureParam.getFirst().getBytes(), "AES");
aesEncryptCipher.init(Cipher.DECRYPT_MODE, secretKeySpec, new IvParameterSpec(secureParam.getSecond().getBytes()));
byte[] compressed = aesEncryptCipher.doFinal(encrypt);
byte[] plainText = decompress(compressed);
result = true;
fileOutputStream.write(plainText);
fileOutputStream.flush();
} catch (Exception e) {
LOGGER.error(e);
}
return result;
}

private static byte[] decompress(byte[] contentBytes) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
IOUtils.copy(new GZIPInputStream(new ByteArrayInputStream(contentBytes)), out);
return out.toByteArray();
} catch (IOException e) {
LOGGER.error(e);
}
return new byte[0];
}

@PreDestroy
public void closeFileSteam() {
try {
fileOutputStream.close();
} catch (IOException e) {
LOGGER.error(e);
}
}

/**
* BouncyCastle作为安全提供，防止我们加密解密时候因为jdk内置的不支持改模式运行报错。
**/
private static void initialize() {
if (initialized.get()) {
return;
}
Security.addProvider(new BouncyCastleProvider());
initialized.set(true);
}


private static Tuple<String, String> getSecureParam() {
try {
Properties properties = PropertiesLoaderUtils.loadAllProperties("secure.properties");
Tuple<String, String> tuple = new Tuple<>();
tuple.setFirst(properties.getProperty("AES_KEY"));
tuple.setSecond(properties.getProperty("IV"));
return tuple;
} catch (IOException e) {
LOGGER.error(e);
}
return null;
}
}
106 changes: 85 additions & 21 deletions Logan/Server/src/main/java/com/meituan/logan/web/parser/LoganProtocol.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
package com.meituan.logan.web.parser;

import com.meituan.logan.web.enums.ResultEnum;
import com.meituan.logan.web.model.Tuple;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.springframework.core.io.support.PropertiesLoaderUtils;

import javax.annotation.PreDestroy;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
Expand All @@ -27,14 +30,17 @@ public class LoganProtocol {

private static final char ENCRYPT_CONTENT_START = '\1';

private static final String AES_ALGORITHM_TYPE = "AES/CBC/NoPadding";
private static final String AES_NO_PADDING = "AES/CBC/NoPadding";
private static final String AES_WITH_PADDING = "AES/CBC/PKCS5Padding";

private static AtomicBoolean initialized = new AtomicBoolean(false);

static {
initialize();
}

private byte[] secretKey;
private byte[] iv;
private ByteBuffer wrap;
private FileOutputStream fileOutputStream;

Expand All @@ -48,8 +54,11 @@ public LoganProtocol(InputStream stream, File file) {
}

public ResultEnum process() {
if (!initSecureParams()) {
return ResultEnum.ERROR_DECRYPT;
}
while (wrap.hasRemaining()) {
while (wrap.get() == ENCRYPT_CONTENT_START) {
while (wrap.hasRemaining() && wrap.get() == ENCRYPT_CONTENT_START) {
byte[] encrypt = new byte[wrap.getInt()];
if (!tryGetEncryptContent(encrypt) || !decryptAndAppendFile(encrypt)) {
return ResultEnum.ERROR_DECRYPT;
Expand All @@ -72,32 +81,47 @@ private boolean tryGetEncryptContent(byte[] encrypt) {
private boolean decryptAndAppendFile(byte[] encrypt) {
boolean result = false;
try {
Cipher aesEncryptCipher = Cipher.getInstance(AES_ALGORITHM_TYPE);
Tuple<String, String> secureParam = getSecureParam();
if (secureParam == null) {
return false;
SecretKeySpec secretKeySpec = new SecretKeySpec(secretKey, "AES");
IvParameterSpec ivParamSpec = new IvParameterSpec(iv);

Cipher cipher;
try {
// 先尝试带 padding 模式解密末尾 16 字节
cipher = Cipher.getInstance(AES_WITH_PADDING);
cipher.init(Cipher.DECRYPT_MODE, secretKeySpec, ivParamSpec);
cipher.doFinal(encrypt, encrypt.length - 16, 16);
} catch (BadPaddingException e) {
LOGGER.warn("decrypt with padding mode fail", e);
// 带 padding 模式解密失败，尝试无 padding 模式
cipher = Cipher.getInstance(AES_NO_PADDING);
}
SecretKeySpec secretKeySpec = new SecretKeySpec(secureParam.getFirst().getBytes(), "AES");
aesEncryptCipher.init(Cipher.DECRYPT_MODE, secretKeySpec, new IvParameterSpec(secureParam.getSecond().getBytes()));
byte[] compressed = aesEncryptCipher.doFinal(encrypt);

cipher.init(Cipher.DECRYPT_MODE, secretKeySpec, ivParamSpec);
byte[] compressed = cipher.doFinal(encrypt);
byte[] plainText = decompress(compressed);
result = true;
fileOutputStream.write(plainText);
fileOutputStream.flush();
} catch (Exception e) {
LOGGER.error(e);
LOGGER.error("decryptAndAppendFile", e);
}
return result;
}

private static byte[] decompress(byte[] contentBytes) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
try {
IOUtils.copy(new GZIPInputStream(new ByteArrayInputStream(contentBytes)), out);
return out.toByteArray();
} catch (IOException e) {
LOGGER.error(e);
// 虽然解压抛了异常，但前面已经解出来的内容还是可用的
// 由于多条日志使用 \n 分割，这里取最后一个 \n 前的内容
byte[] arr = out.toByteArray();
int lastIndexOfLf = ArrayUtils.lastIndexOf(arr, (byte) '\n');
arr = lastIndexOfLf < 0 ? new byte[0] : ArrayUtils.subarray(arr, 0, lastIndexOfLf + 1);
LOGGER.error("decompress, dropped=" + (out.size() - arr.length), e);
return arr;
}
return new byte[0];
}

@PreDestroy
Expand All @@ -121,16 +145,56 @@ private static void initialize() {
}


private static Tuple<String, String> getSecureParam() {
private boolean initSecureParams() {
if (checkSecureParams()) {
return true;
}
try {
Properties properties = PropertiesLoaderUtils.loadAllProperties("secure.properties");
Tuple<String, String> tuple = new Tuple<>();
tuple.setFirst(properties.getProperty("AES_KEY"));
tuple.setSecond(properties.getProperty("IV"));
return tuple;
secretKey = properties.getProperty("AES_KEY").getBytes();
iv = properties.getProperty("IV").getBytes();
return checkSecureParams();
} catch (IOException e) {
LOGGER.error(e);
LOGGER.error("initSecureParams", e);
}
return false;
}

private boolean checkSecureParams() {
return secretKey != null && secretKey.length == 16 &&
iv != null && iv.length == 16;
}

private static int countLines(File file) throws IOException {
int count = 0;
try (LineIterator iterator = FileUtils.lineIterator(file)) {
while (iterator.hasNext()) {
count++;
iterator.next();
}
}
return count;
}

public static void main(String[] args) throws IOException {
if (args.length < 2) {
System.exit(-1);
}
File input = new File(args[0]); // 原始文件路径
File output = new File(args[1]); // 输出文件路径
try (FileInputStream inputStream = new FileInputStream(input)) {
LoganProtocol protocol = new LoganProtocol(inputStream, output);
ResultEnum result = protocol.process();
System.out.println("result: " + result);
}
System.out.println("output lines: " + countLines(output));

output = new File(output.getAbsolutePath() + ".legacy");
try (FileInputStream inputStream = new FileInputStream(input)) {
LegacyLoganProtocol protocol = new LegacyLoganProtocol(inputStream, output);
ResultEnum result = protocol.process();
System.out.println("legacy result: " + result);
}
return null;
System.out.println("legacy output lines: " + countLines(output));
}
}

0 comments on commit 3a81b56

Please sign in to comment.