Skip to content

Commit 4d93f6f

Browse files
nnhynnhy
committed
[feat] v3.3 星尘监控表采用循环天表，一次性创建好31张表，无需每天自动创建新表
1 parent ff69847 commit 4d93f6f

File tree

11 files changed

+194
-98
lines changed

11 files changed

+194
-98
lines changed

‎StarAgent/StarAgent.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<Description>星尘，分布式资源调度，部署于每一个节点，连接服务端，支持节点监控、远程发布。</Description>
88
<Company>新生命开发团队</Company>
99
<Copyright>©2002-2025 NewLife</Copyright>
10-
<VersionPrefix>3.1</VersionPrefix>
10+
<VersionPrefix>3.3</VersionPrefix>
1111
<VersionSuffix>$([System.DateTime]::Now.ToString(`yyyy.MMdd`))</VersionSuffix>
1212
<Version>$(VersionPrefix).$(VersionSuffix)</Version>
1313
<FileVersion>$(Version)</FileVersion>

‎Stardust.Data/Entity/应用日志.Biz.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using System;
1+
using System;
22
using System.Collections.Generic;
33
using System.ComponentModel;
44
using System.Runtime.Serialization;
@@ -21,7 +21,7 @@ static AppClientLog()
2121
Meta.ShardPolicy = new TimeShardPolicy(nameof(Id), Meta.Factory)
2222
{
2323
ConnPolicy = "{0}_{1:yyyyMM}",
24-
TablePolicy = "{0}_{1:yyyyMMdd}",
24+
TablePolicy = "{0}_{1:dd}",
2525
};
2626

2727
// 累加字段，生成 Update xx Set Count=Count+1234 Where xxx

‎Stardust.Data/Monitors/Model.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@
134134
</Table>
135135
<Table Name="TraceData" Description="跟踪数据。应用定时上报采样得到的埋点追踪原始数据，应用端已完成初步统计，后端将再次向上汇总" ConnName="StardustData">
136136
<Columns>
137-
<Column Name="Id" DataType="Int64" PrimaryKey="True" DataScale="timeShard:yyyyMMdd" Description="编号" />
137+
<Column Name="Id" DataType="Int64" PrimaryKey="True" DataScale="timeShard:dd" Description="编号" />
138138
<Column Name="StatDate" DataType="DateTime" Description="统计日期" />
139139
<Column Name="StatHour" DataType="DateTime" Description="统计小时" />
140140
<Column Name="StatMinute" DataType="DateTime" Description="统计分钟" />
@@ -168,7 +168,7 @@
168168
</Table>
169169
<Table Name="SampleData" Description="采样数据。具体调用或异常详情，每次追踪统计携带少量样板，用于链路分析以及异常追踪" ConnName="StardustData">
170170
<Columns>
171-
<Column Name="Id" DataType="Int64" PrimaryKey="True" DataScale="timeShard:yyyyMMdd" Description="编号" />
171+
<Column Name="Id" DataType="Int64" PrimaryKey="True" DataScale="timeShard:dd" Description="编号" />
172172
<Column Name="DataId" DataType="Int64" Description="数据" />
173173
<Column Name="ItemId" DataType="Int32" Description="跟踪项" />
174174
<Column Name="Success" DataType="Boolean" Description="正常" />

‎Stardust.Data/Monitors/跟踪数据.Biz.cs

+3-6
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ static TraceData()
2222
Meta.ShardPolicy = new TimeShardPolicy(nameof(Id), Meta.Factory)
2323
{
2424
ConnPolicy = "{0}",
25-
TablePolicy = "{0}_{1:yyyyMMdd}",
25+
TablePolicy = "{0}_{1:dd}",
2626
Step = TimeSpan.FromDays(1),
2727
};
2828

@@ -371,11 +371,8 @@ public static Int32 DeleteBefore(DateTime date)
371371
var snow = Meta.Factory.Snow;
372372
var whereExp = _.Id < snow.GetId(date);
373373

374-
// 使用底层接口，加大执行时间
375-
var session = Meta.Session;
376-
using var cmd = session.Dal.Session.CreateCommand($"Delete From {session.FormatedTableName} Where {whereExp}");
377-
cmd.CommandTimeout = 5 * 60;
378-
return session.Dal.Session.Execute(cmd);
374+
// 使用底层接口，分批删除
375+
return Delete(whereExp);
379376
}
380377
#endregion
381378
}

‎Stardust.Data/Monitors/采样数据.Biz.cs

+3-6
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ static SampleData()
2323
Meta.ShardPolicy = new TimeShardPolicy(nameof(Id), Meta.Factory)
2424
{
2525
ConnPolicy = "{0}",
26-
TablePolicy = "{0}_{1:yyyyMMdd}",
26+
TablePolicy = "{0}_{1:dd}",
2727
Step = TimeSpan.FromDays(1),
2828
};
2929

@@ -266,11 +266,8 @@ public static Int32 DeleteBefore(DateTime date)
266266
var snow = Meta.Factory.Snow;
267267
var whereExp = _.Id < snow.GetId(date);
268268

269-
// 使用底层接口，加大执行时间
270-
var session = Meta.Session;
271-
using var cmd = session.Dal.Session.CreateCommand($"Delete From {session.FormatedTableName} Where {whereExp}");
272-
cmd.CommandTimeout = 5 * 60;
273-
return session.Dal.Session.Execute(cmd);
269+
// 使用底层接口，分批删除
270+
return Delete(whereExp);
274271
}
275272
#endregion
276273
}

‎Stardust.Extensions/Stardust.Extensions.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<Description>星尘，分布式服务框架扩展。节点管理，监控中心，配置中心，发布中心，注册中心</Description>
88
<Company>新生命开发团队</Company>
99
<Copyright>©2002-2025 NewLife</Copyright>
10-
<VersionPrefix>3.2</VersionPrefix>
10+
<VersionPrefix>3.3</VersionPrefix>
1111
<VersionSuffix>$([System.DateTime]::Now.ToString(`yyyy.MMdd`))</VersionSuffix>
1212
<Version>$(VersionPrefix).$(VersionSuffix)</Version>
1313
<FileVersion>$(Version)</FileVersion>

‎Stardust.Server/Services/ShardTableService.cs

+173-73
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ private void DoShardTable(Object state)
4040

4141
// 保留数据的起点
4242
var today = DateTime.Today;
43-
var endday = today.AddDays(-_setting.DataRetention);
43+
var days = _setting.DataRetention;
44+
var endday = today.AddDays(-days);
4445

4546
XTrace.WriteLine("检查数据分表，保留数据起始日期：{0:yyyy-MM-dd}", endday);
4647

@@ -49,107 +50,106 @@ private void DoShardTable(Object state)
4950
{
5051
// 取所有表，清空缓存
5152
var dal = TraceData.Meta.Session.Dal;
52-
var dal2 = AppTracer.Meta.Session.Dal;
5353

5454
dal.Tables = null;
5555
var tables = dal.Tables;
5656
var tnames = tables.Select(e => e.TableName).ToArray();
5757

58-
for (var dt = today.AddYears(-10); dt < endday; dt = dt.AddDays(1))
58+
// 检查表结构
59+
var ts = new List<IDataTable>();
60+
for (var i = 0; i < 31; i++)
5961
{
60-
var name = $"SampleData_{dt:yyyyMMdd}";
61-
if (name.EqualIgnoreCase(tnames))
62+
var date = today.AddDays(1 - i);
63+
6264
{
63-
try
64-
{
65-
dal.Execute($"Drop Table {name}");
66-
}
67-
catch (Exception ex)
68-
{
69-
XTrace.WriteException(ex);
70-
}
65+
var table = TraceData.Meta.Table.DataTable.Clone() as IDataTable;
66+
table.TableName = $"TraceData_{date:dd}";
67+
ts.Add(table);
7168
}
72-
73-
name = $"TraceData_{dt:yyyyMMdd}";
74-
if (name.EqualIgnoreCase(tnames))
7569
{
76-
try
77-
{
78-
dal.Execute($"Drop Table {name}");
79-
}
80-
catch (Exception ex)
81-
{
82-
XTrace.WriteException(ex);
83-
}
70+
var table = SampleData.Meta.Table.DataTable.Clone() as IDataTable;
71+
table.TableName = $"SampleData_{date:dd}";
72+
ts.Add(table);
8473
}
8574
}
8675

87-
// 数据迁移后，原库数据表需要清理。需要重新获取表名列表，因为Stardust/StardustData可能指向同一个数据库
88-
dal2.Tables = null;
89-
var tnames2 = dal2.Tables.Select(e => e.TableName).ToArray();
90-
for (var dt = today.AddYears(-10); dt < endday; dt = dt.AddDays(1))
76+
if (ts.Count > 0)
9177
{
92-
var name = $"SampleData_{dt:yyyyMMdd}";
93-
if (name.EqualIgnoreCase(tnames2))
94-
{
95-
try
96-
{
97-
dal2.Execute($"Drop Table {name}");
98-
}
99-
catch { }
100-
}
78+
XTrace.WriteLine("检查循环天表[{0}]：{1}", ts.Count, ts.Join(",", e => e.TableName));
79+
80+
//dal.SetTables(ts.ToArray());
81+
dal.Db.CreateMetaData().SetTables(Migration.On, ts.ToArray());
10182

102-
name = $"TraceData_{dt:yyyyMMdd}";
103-
if (name.EqualIgnoreCase(tnames2))
83+
// 首次建表时，设置为压缩表
84+
if (dal.DbType == DatabaseType.MySql)
10485
{
105-
try
86+
foreach (var dt in ts)
10687
{
107-
dal2.Execute($"Drop Table {name}");
88+
dal.Execute($"Alter Table {dt.TableName} ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4");
10889
}
109-
catch { }
11090
}
11191
}
11292

113-
// 新建今天明天的表
114-
var ts = new List<IDataTable>();
115-
{
116-
var table = TraceData.Meta.Table.DataTable.Clone() as IDataTable;
117-
table.TableName = $"TraceData_{today:yyyyMMdd}";
118-
ts.Add(table);
119-
}
120-
{
121-
var table = TraceData.Meta.Table.DataTable.Clone() as IDataTable;
122-
table.TableName = $"TraceData_{today.AddDays(1):yyyyMMdd}";
123-
ts.Add(table);
124-
}
125-
{
126-
var table = SampleData.Meta.Table.DataTable.Clone() as IDataTable;
127-
table.TableName = $"SampleData_{today:yyyyMMdd}";
128-
ts.Add(table);
129-
}
93+
94+
// 如果保留时间超过了31天，则使用删除功能清理历史数据，否则使用truncate
95+
if (days > 31)
13096
{
131-
var table = SampleData.Meta.Table.DataTable.Clone() as IDataTable;
132-
table.TableName = $"SampleData_{today.AddDays(1):yyyyMMdd}";
133-
ts.Add(table);
97+
// 31张表里面，每张表都删除指定时间之前的数据
98+
for (var i = 0; i < 31; i++)
99+
{
100+
TraceData.DeleteBefore(endday);
101+
SampleData.DeleteBefore(endday);
102+
}
134103
}
135-
136-
if (ts.Count > 0)
104+
else
137105
{
138-
XTrace.WriteLine("创建或更新数据表[{0}]：{1}", ts.Count, ts.Join(",", e => e.TableName));
106+
using var showSql = dal.Session.SetShowSql(true);
139107

140-
//dal.SetTables(ts.ToArray());
141-
dal.Db.CreateMetaData().SetTables(Migration.On, ts.ToArray());
142-
143-
// 首次建表时，设置为压缩表
144-
if (dal.DbType == DatabaseType.MySql)
108+
// 遍历31张表，只要大于结束时间则安全，否则清空
109+
for (var i = 0; i < 31; i++)
145110
{
146-
foreach (var dt in ts)
111+
var dt = today.AddDays(-i);
112+
if (dt >= endday) continue;
113+
114+
var name = $"TraceData_{dt:dd}";
115+
if (name.EqualIgnoreCase(tnames))
116+
{
117+
try
118+
{
119+
if (dal.DbType == DatabaseType.SQLite)
120+
TraceData.DeleteBefore(endday);
121+
else
122+
dal.Execute($"Truncate Table {name}");
123+
//dal.Session.Truncate(name);
124+
}
125+
catch (Exception ex)
126+
{
127+
XTrace.WriteException(ex);
128+
}
129+
}
130+
name = $"SampleData_{dt:dd}";
131+
if (name.EqualIgnoreCase(tnames))
147132
{
148-
if (!dt.TableName.EqualIgnoreCase(tnames))
149-
dal.Execute($"Alter Table {dt.TableName} ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4");
133+
try
134+
{
135+
if (dal.DbType == DatabaseType.SQLite)
136+
SampleData.DeleteBefore(endday);
137+
else
138+
dal.Execute($"Truncate Table {name}");
139+
//dal.Session.Truncate(name);
140+
}
141+
catch (Exception ex)
142+
{
143+
XTrace.WriteException(ex);
144+
}
150145
}
151146
}
152147
}
148+
149+
// 数据迁移后，原库数据表需要清理。需要重新获取表名列表，因为Stardust/StardustData可能指向同一个数据库
150+
DropOldTable(dal);
151+
var dal2 = AppTracer.Meta.Session.Dal;
152+
DropOldTable(dal2);
153153
}
154154
catch (Exception ex)
155155
{
@@ -159,4 +159,104 @@ private void DoShardTable(Object state)
159159

160160
XTrace.WriteLine("检查数据表完成");
161161
}
162+
163+
static void DropOldTable(DAL dal)
164+
{
165+
using var showSql = dal.Session.SetShowSql(true);
166+
167+
dal.Tables = null;
168+
var tnames2 = dal.Tables.Select(e => e.TableName).ToArray();
169+
var today = DateTime.Today;
170+
for (var dt = today.AddYears(-10); dt <= today; dt = dt.AddDays(1))
171+
{
172+
var name = $"SampleData_{dt:yyyyMMdd}";
173+
if (name.EqualIgnoreCase(tnames2))
174+
{
175+
try
176+
{
177+
dal.Execute($"Drop Table {name}");
178+
}
179+
catch { }
180+
}
181+
182+
name = $"TraceData_{dt:yyyyMMdd}";
183+
if (name.EqualIgnoreCase(tnames2))
184+
{
185+
try
186+
{
187+
dal.Execute($"Drop Table {name}");
188+
}
189+
catch { }
190+
}
191+
}
192+
}
193+
194+
/// <summary>修正自动分表。20250103启用新的循环天表</summary>
195+
public static void FixShardTable()
196+
{
197+
var dal = TraceData.Meta.Session.Dal;
198+
var tables = dal.Tables;
199+
if (tables == null) return;
200+
201+
using var showSql = dal.Session.SetShowSql(true);
202+
203+
// 从明天起倒推31天，保证31张表，如果旧表则重命名
204+
var today = DateTime.Today;
205+
206+
// 新建缺失表
207+
//var ts = new List<IDataTable>();
208+
for (var i = 0; i < 31; i++)
209+
{
210+
var date = today.AddDays(1 - i);
211+
212+
var newName = $"TraceData_{date:dd}";
213+
var table = tables.FirstOrDefault(e => e.TableName.EqualIgnoreCase(newName));
214+
if (table == null)
215+
{
216+
var oldName = $"TraceData_{date:yyyyMMdd}";
217+
table = tables.FirstOrDefault(e => e.TableName.EqualIgnoreCase(oldName));
218+
if (table != null)
219+
dal.Execute($"Alter Table {table.TableName} Rename To {newName}");
220+
//else
221+
//{
222+
// table = TraceData.Meta.Table.DataTable.Clone() as IDataTable;
223+
// table.TableName = newName;
224+
// ts.Add(table);
225+
//}
226+
}
227+
228+
newName = $"SampleData_{date:dd}";
229+
table = tables.FirstOrDefault(e => e.TableName.EqualIgnoreCase(newName));
230+
if (table == null)
231+
{
232+
var oldName = $"SampleData_{date:yyyyMMdd}";
233+
table = tables.FirstOrDefault(e => e.TableName.EqualIgnoreCase(oldName));
234+
if (table != null)
235+
dal.Execute($"Alter Table {table.TableName} Rename To {newName}");
236+
//else
237+
//{
238+
// table = SampleData.Meta.Table.DataTable.Clone() as IDataTable;
239+
// table.TableName = newName;
240+
// ts.Add(table);
241+
//}
242+
}
243+
}
244+
245+
//if (ts.Count > 0)
246+
//{
247+
// XTrace.WriteLine("迁移到循环天表[{0}]：{1}", ts.Count, ts.Join(",", e => e.TableName));
248+
249+
// //dal.SetTables(ts.ToArray());
250+
// dal.Db.CreateMetaData().SetTables(Migration.On, ts.ToArray());
251+
252+
// // 首次建表时，设置为压缩表
253+
// if (dal.DbType == DatabaseType.MySql)
254+
// {
255+
// foreach (var dt in ts)
256+
// {
257+
// dal.Execute($"Alter Table {dt.TableName} ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4");
258+
// }
259+
// }
260+
//}
261+
}
162262
}

0 commit comments

Comments
 (0)