Skip to content

Commit

Permalink
feat: 分表,数据保留天数
Browse files Browse the repository at this point in the history
eddie murphy committed Sep 21, 2024
1 parent 2bc8243 commit 338eca0
Showing 12 changed files with 118 additions and 34 deletions.
14 changes: 1 addition & 13 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
version=$(shell cat VERSION)
image_name="wangduanduan/sipgrep-go:$(version)"

DBAddr=
DBName=
DBUserPasswd=

dev:
watchexec -r -e mjs -- bun app.mjs
@@ -16,13 +13,4 @@ image-push:
release:
git tag v$(version)
git push origin v$(version)
changelog:
git-chglog -o CHANGELOG.md
t1:
http --verbose localhost:3000/api/v1/call BeginTime=="2023-11-05 00:00:00" EndTime=="2023-11-05 23:59:59"
t2:
http --verbose localhost:3000/api/v1/call/2023-11-11/oMDfqeY4EHHwovasP2Mn9x3aFzOy6Lvw
t3:
http --verbose localhost:3000/api/v1/2023-11-12/ccgLi7C.C6PxAqJWT-RrUyWk6MI0BZJq
t4:
http --verbose localhost:3000/api/v1/call/2023-11-12/25ya3ru5Kx2TJDBrXYMreSlBcsuCLFxL/

7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -25,8 +25,8 @@

- [x] 时序图搜索
- [x] 时序图展示
- [ ] 分表
- [ ] 数据保留日期设置
- [x] 分表
- [x] 数据保留天数设置
- [ ] 收藏
- [ ] 导入pcap
- [ ] 导入json
@@ -48,6 +48,7 @@ docker run -d --name=siphub \
-e DBAddr=1.2.3.4 \
-e DBPort=5432 \
-e DBName=postgres \
-e dataKeepDays=10 \
-p 3000:3000 \
eddiemurphy5/siphub:latest
```
@@ -61,7 +62,7 @@ docker run -d --name=siphub \
- DBName: 数据库名,默认postgres,
- LogLevel: 日志级别, 默认debug
- QueryLimit: 一次性查询的行数,默认10

- dataKeepDays: 数据保留几天,默认3

# 架构图

9 changes: 9 additions & 0 deletions app.mjs
Original file line number Diff line number Diff line change
@@ -4,6 +4,8 @@ import { AppLoger, logger } from './logger.mjs'
import dayjs from 'dayjs'
import { route } from './router/api.mjs'
import { queryRecord } from './db.mjs'
import { startCron } from './cron.mjs'
import { AppEnv } from './env.mjs'

const app = express()

@@ -12,6 +14,13 @@ app.use(bodyParser.urlencoded({ extended: false }))
app.use(express.static('public'))
app.set('view engine', 'ejs')

if (AppEnv.enableCron === 'yes') {
logger.info('enable crontab')
startCron()
} else {
logger.info('disable crontab')
}

app.get('/', async function (req, res) {
let n = dayjs()
let day = n.format('YYYY-MM-DD')
20 changes: 20 additions & 0 deletions cron.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { CronJob } from 'cron';
import { AppEnv } from './env.mjs';
import { logger } from './logger.mjs';
import { tableSplit, deleteTable } from './db.mjs';

export function startCron() {
const job = new CronJob(
AppEnv.cronTime,
async function () {
logger.info('cron start')
await tableSplit()
await deleteTable()
},
function () {
logger.info('cron complete')
},
true,
AppEnv.timeZone
);
}
61 changes: 51 additions & 10 deletions db.mjs
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ import { AppEnv } from './env.mjs'
import { logger } from './logger.mjs'
import { whereBuilder } from './util.mjs'
const { Client, Pool } = pg
import dayjs from 'dayjs'

const pool = new Pool({
user: AppEnv.DBUser,
@@ -15,16 +16,58 @@ const pool = new Pool({
connectionTimeoutMillis: 2000,
})

function getTableNameByDay(day) {
let today = dayjs().format('YYYY-MM-DD')
if (day === today) {
return 'records'
}

return `records_${day.replaceAll('-', '')}`
}

export async function tableSplit() {
let tableDay = dayjs().subtract(1, 'day').format("YYYYMMDD")

const sql = `
CREATE table if not exists records_tmp (LIKE public.records INCLUDING all);
ALTER TABLE records RENAME TO records_${tableDay};
ALTER TABLE records_tmp RENAME TO records;
`

logger.info(sql)

return await pool.query(sql)
}

export async function deleteTable() {
let res = await pool.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public' and
table_name like 'records_%'
order by table_name desc
offset ${AppEnv.dataKeepDays};
`)

if (res.rows.length === 0) {
logger.info("没有需要删除的表")
}

for (const ele of res.rows) {
console.log(ele.table_name)
logger.info(`try delete table ${ele.table_name}`)
await pool.query(`DROP TABLE IF EXISTS ${ele.table_name}`)
}
}

export async function queryRecord(c) {
logger.info(c)
console.log(c)
let wh = whereBuilder(c)
const sql = `
select
sip_call_id as "CallID",
to_char(min(create_time),'HH24:MI:SS') as "startTime",
to_char(min(create_time),'YYYY-MM-DD') as "day",
to_char(max(create_time),'HH24:MI:SS') as "stopTime",
to_char(max(create_time) - min(create_time),'HH24:MI:SS') as "duration",
min(from_user) as "caller",
@@ -34,7 +77,7 @@ export async function queryRecord(c) {
max(response_code)::int as "finalCode",
string_agg(DISTINCT CASE WHEN response_code BETWEEN 170 AND 190 THEN response_code::text END, ',') AS "tempCode"
from
public.records
public.${getTableNameByDay(c.day)}
where
${wh.join(' and ')}
group by sip_call_id
@@ -52,7 +95,7 @@ export async function queryRecord(c) {

export async function queryById(id, day) {
const sql = `
select
select
sip_call_id,
sip_method,
to_char(create_time,
@@ -71,13 +114,11 @@ export async function queryById(id, day) {
replace(dst_host,':','_') as dst_host,
response_desc,
length(raw_msg) as msg_len
from
public.records
where
sip_call_id = '${id}'
order by
create_time ,
timestamp_micro
from
public.${getTableNameByDay(day)}
where
sip_call_id = '${id}'
order by create_time , timestamp_micro
`

logger.info(sql)
6 changes: 5 additions & 1 deletion env.mjs
Original file line number Diff line number Diff line change
@@ -5,5 +5,9 @@ export const AppEnv = {
DBPort: process.env.DBPort ? parseInt(process.env.DBPort) : 5432,
DBName: process.env.DBName ?? 'postgres',
LogLevel: process.env.LogLevel ?? 'debug',
QueryLimit: process.env.QueryLimit ? parseInt(process.env.QueryLimit) : 10
QueryLimit: process.env.QueryLimit ? parseInt(process.env.QueryLimit) : 10,
cronTime: process.env.cronTime ?? '0 0 0 * * *',
timeZone: process.env.timeZone ?? 'Asia/Shanghai',
enableCron: process.env.enableCron ?? 'yes',
dataKeepDays: process.env.dataKeepDays ? parseInt(process.env.dataKeepDays) : 3
}
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
},
"dependencies": {
"body-parser": "^1.20.2",
"cron": "^3.1.7",
"dayjs": "^1.11.13",
"ejs": "^3.1.10",
"express": "^4.19.2",
22 changes: 22 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 2 additions & 4 deletions router/api.mjs
Original file line number Diff line number Diff line change
@@ -12,12 +12,10 @@ route.post('/record', async (req, res) => {
})

route.get('/call', async (req, res) => {
let re = await queryById(req.query.id)
// res.render('home/sipcdr', { table: re.rows })
logger.info(re.rows)
let re = await queryById(req.query.id, req.query.day)
let rows = re.rows
let seq = createSeqHtml(rows)
logger.info(seq)
logger.debug(seq)

res.render('diagram/index', {
seq: seq.html, table: rows
2 changes: 1 addition & 1 deletion util.mjs
Original file line number Diff line number Diff line change
@@ -95,4 +95,4 @@ export function createSeqHtml(seq) {
return {
html: res.join('\n'),
}
}
}
2 changes: 1 addition & 1 deletion views/diagram/index.ejs
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@
<body>
<div class="container-fluid">
<div class="row">
<div class="col-md-12">1</div>
<div class="col-md-12"></div>
</div>
<div class="row">
<div class="col-md-8" style="max-height: 800px;overflow-y: scroll;">
2 changes: 1 addition & 1 deletion views/home/sipcdr.ejs
Original file line number Diff line number Diff line change
@@ -24,7 +24,7 @@
<td><%= v.stopTime %></td>
<td><%= v.duration %></td>
<td>
<a href="/api/call?id=<%= v.CallID %>&day=<%= v.startTime %>" target="_blank"> <%= v.CallID %> </a>
<a href="/api/call?id=<%= v.CallID %>&day=<%= v.day %>" target="_blank"> <%= v.CallID %> </a>
</td>
<td><%= v.caller %></td>
<td><%= v.callee %></td>

0 comments on commit 338eca0

Please sign in to comment.